From c18f53b8ead0652db9dd473eeb23ccf32e79508d Mon Sep 17 00:00:00 2001 From: genofire Date: Mon, 26 Oct 2020 23:46:16 +0100 Subject: [PATCH] Add rspamd support. Postfix uses the Rspamd proxy as spam filtering milter and HAProxy exposes the Rspamd webinterface through HTTPS. Updates #10 --- mail.yml | 1 + roles/rspamd/handlers/main.yml | 20 +++ roles/rspamd/meta/main.yml | 13 ++ roles/rspamd/tasks/main.yml | 121 ++++++++++++++++++ .../local.d/classifier-bayes.conf.j2 | 4 + .../templates/local.d/composites.conf.j2 | 8 ++ .../templates/local.d/fann_redis.conf.j2 | 3 + roles/rspamd/templates/local.d/options.inc.j2 | 5 + roles/rspamd/templates/local.d/redis.conf.j2 | 3 + .../templates/override.d/logging.inc.j2 | 5 + .../override.d/worker-controller.inc.j2 | 6 + .../templates/override.d/worker-fuzzy.inc.j2 | 4 + .../templates/override.d/worker-normal.inc.j2 | 3 + .../templates/override.d/worker-proxy.inc.j2 | 10 ++ roles/rspamd/templates/rspamd-log/finish.j2 | 13 ++ roles/rspamd/templates/rspamd-log/run.j2 | 23 ++++ roles/rspamd/templates/rspamd.cfg.j2 | 23 ++++ roles/rspamd/templates/rspamd/data/check.j2 | 19 +++ roles/rspamd/templates/rspamd/finish.j2 | 13 ++ roles/rspamd/templates/rspamd/run.j2 | 13 ++ roles/rspamd/vars/main.yml | 71 ++++++++++ 21 files changed, 381 insertions(+) create mode 100644 roles/rspamd/handlers/main.yml create mode 100644 roles/rspamd/meta/main.yml create mode 100644 roles/rspamd/tasks/main.yml create mode 100644 roles/rspamd/templates/local.d/classifier-bayes.conf.j2 create mode 100644 roles/rspamd/templates/local.d/composites.conf.j2 create mode 100644 roles/rspamd/templates/local.d/fann_redis.conf.j2 create mode 100644 roles/rspamd/templates/local.d/options.inc.j2 create mode 100644 roles/rspamd/templates/local.d/redis.conf.j2 create mode 100644 roles/rspamd/templates/override.d/logging.inc.j2 create mode 100644 roles/rspamd/templates/override.d/worker-controller.inc.j2 create mode 100644 roles/rspamd/templates/override.d/worker-fuzzy.inc.j2 create mode 100644 roles/rspamd/templates/override.d/worker-normal.inc.j2 create mode 100644 roles/rspamd/templates/override.d/worker-proxy.inc.j2 create mode 100644 roles/rspamd/templates/rspamd-log/finish.j2 create mode 100644 roles/rspamd/templates/rspamd-log/run.j2 create mode 100644 roles/rspamd/templates/rspamd.cfg.j2 create mode 100644 roles/rspamd/templates/rspamd/data/check.j2 create mode 100644 roles/rspamd/templates/rspamd/finish.j2 create mode 100644 roles/rspamd/templates/rspamd/run.j2 create mode 100644 roles/rspamd/vars/main.yml diff --git a/mail.yml b/mail.yml index 5617c18..752d026 100644 --- a/mail.yml +++ b/mail.yml @@ -17,4 +17,5 @@ roles: - dovecot + - rspamd - postfix diff --git a/roles/rspamd/handlers/main.yml b/roles/rspamd/handlers/main.yml new file mode 100644 index 0000000..1f29368 --- /dev/null +++ b/roles/rspamd/handlers/main.yml @@ -0,0 +1,20 @@ +--- +- name: Reload s6-rc + service: + name: s6-rc + state: reloaded + +- name: Restart Rspamd + command: s6-svc -wR -T 5000 -ru /run/service/rspamd + +- name: Restart Rspamd log + command: s6-svc -wR -T 5000 -ru /run/service/rspamd-log + +- name: Restart HAProxy + command: s6-svc -wU -T 5000 -ru /run/service/haproxy + +- name: Restart HAProxy log + command: s6-svc -wU -T 5000 -ru /run/service/haproxy-log + +- name: Reload HAProxy + command: s6-svc -2 /run/service/haproxy diff --git a/roles/rspamd/meta/main.yml b/roles/rspamd/meta/main.yml new file mode 100644 index 0000000..7680923 --- /dev/null +++ b/roles/rspamd/meta/main.yml @@ -0,0 +1,13 @@ +--- +#dependencies: +# - role: redis +# redis_instance: other +# redis_client_group: '236' +# +# - role: redis +# redis_instance: bayes +# redis_client_group: '236' +# +# - role: redis +# redis_instance: fuzzy +# redis_client_group: '236' diff --git a/roles/rspamd/tasks/main.yml b/roles/rspamd/tasks/main.yml new file mode 100644 index 0000000..bf4a244 --- /dev/null +++ b/roles/rspamd/tasks/main.yml @@ -0,0 +1,121 @@ +--- +- name: Install Rspamd + package: + name: rspamd + +- name: Create rspamd local.d and override.d + file: + path: '/usr/local/etc/rspamd/{{ item }}' + state: directory + mode: 0755 + owner: root + group: wheel + with_items: + - local.d + - override.d + +- name: Configure rspamd + template: + dest: '/usr/local/etc/rspamd/{{ item }}' + src: '{{ item }}.j2' + mode: 0444 + owner: root + group: wheel + notify: + - Restart Rspamd + with_items: '{{ rspamd_config }}' + +- name: Add /var/log/rspamd to fstab + mount: + path: /var/log/rspamd + src: tmpfs + fstype: tmpfs + opts: 'rw,size={{ rspamd_log_size }},mode={{ rspamd_log_mode }},uid={{ rspamd_log_uid }},gid={{ rspamd_log_gid }},late' + state: mounted + +- name: Create rspamd HAProxy socket directory + file: + path: /var/run/haproxy/rspamd + state: directory + owner: haproxy + group: rspamd + mode: 0770 + +- name: Create Rspamd service directories + file: + path: '/etc/s6-rc/service/{{ item }}' + state: directory + owner: root + group: wheel + mode: 0755 + with_items: '{{ rspamd_service_dirs }}' + notify: + - Reload s6-rc + - Restart Rspamd log + - Restart Rspamd + +- name: Generate Rspamd service scripts + template: + dest: '/etc/s6-rc/service/{{ item }}' + src: '{{ item }}.j2' + mode: 0555 + owner: root + group: wheel + with_items: '{{ rspamd_service_scripts }}' + notify: + - Reload s6-rc + - Restart Rspamd log + - Restart Rspamd + +- name: Generate Rspamd service configuration + copy: + dest: '/etc/s6-rc/service/{{ item.name }}' + content: '{{ item.content }}' + mode: 0444 + owner: root + group: wheel + loop_control: + label: '{{ item.name }} = {{ item.content }}' + notify: + - Reload s6-rc + - Restart Rspamd log + - Restart Rspamd + with_items: '{{ rspamd_service_config }}' + +- name: Flush handlers + meta: flush_handlers + +- name: Start Rspamd + command: fdmove -c 2 1 s6-rc -u -v 2 -t 15000 change rspamd + register: change + changed_when: change.stdout | length > 0 + +- name: Enable Rspamd + lineinfile: + path: /etc/s6-rc/service/enabled/contents + regexp: "^rspamd$" + line: rspamd + state: present + notify: + - Reload s6-rc + +- name: Flush handlers (again) + meta: flush_handlers + +- name: Tell HAProxy where to find the Rspamd webinterface + template: + dest: /usr/local/etc/haproxy/rspamd.cfg + src: rspamd.cfg.j2 + owner: root + group: wheel + mode: 0444 + notify: + - Reload HAProxy + +- name: Expose the Rspamd webinterface via HTTPS + lineinfile: + path: /usr/local/etc/haproxy/sni.map + regexp: '^{{ ansible_fqdn }} ' + line: '{{ ansible_fqdn }} sni_rspamd' + notify: + - Reload HAProxy diff --git a/roles/rspamd/templates/local.d/classifier-bayes.conf.j2 b/roles/rspamd/templates/local.d/classifier-bayes.conf.j2 new file mode 100644 index 0000000..37479c3 --- /dev/null +++ b/roles/rspamd/templates/local.d/classifier-bayes.conf.j2 @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +backend = "redis"; +servers = "/var/run/redis-bayes/sock"; diff --git a/roles/rspamd/templates/local.d/composites.conf.j2 b/roles/rspamd/templates/local.d/composites.conf.j2 new file mode 100644 index 0000000..74c8570 --- /dev/null +++ b/roles/rspamd/templates/local.d/composites.conf.j2 @@ -0,0 +1,8 @@ +# {{ ansible_managed }} + +MAILLIST_FROM_PHPMAILER { + expression = "MAILLIST & HAS_PHPMAILER_SIG"; + description = "PHPMailer sending mail to a mailing-list"; + score = 20; + policy = "leave"; +} diff --git a/roles/rspamd/templates/local.d/fann_redis.conf.j2 b/roles/rspamd/templates/local.d/fann_redis.conf.j2 new file mode 100644 index 0000000..ec61e42 --- /dev/null +++ b/roles/rspamd/templates/local.d/fann_redis.conf.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} + +timeout = 60s; diff --git a/roles/rspamd/templates/local.d/options.inc.j2 b/roles/rspamd/templates/local.d/options.inc.j2 new file mode 100644 index 0000000..61ad142 --- /dev/null +++ b/roles/rspamd/templates/local.d/options.inc.j2 @@ -0,0 +1,5 @@ +# {{ ansible_managed }} + +dns { + nameserver = ["[::1]"]; +} diff --git a/roles/rspamd/templates/local.d/redis.conf.j2 b/roles/rspamd/templates/local.d/redis.conf.j2 new file mode 100644 index 0000000..eaf48e7 --- /dev/null +++ b/roles/rspamd/templates/local.d/redis.conf.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} + +servers = "/var/run/redis-other/sock"; diff --git a/roles/rspamd/templates/override.d/logging.inc.j2 b/roles/rspamd/templates/override.d/logging.inc.j2 new file mode 100644 index 0000000..be08070 --- /dev/null +++ b/roles/rspamd/templates/override.d/logging.inc.j2 @@ -0,0 +1,5 @@ +# {{ ansible_managed }} + +type = "console"; +systemd = true; +log_color = false; diff --git a/roles/rspamd/templates/override.d/worker-controller.inc.j2 b/roles/rspamd/templates/override.d/worker-controller.inc.j2 new file mode 100644 index 0000000..ae1c0c7 --- /dev/null +++ b/roles/rspamd/templates/override.d/worker-controller.inc.j2 @@ -0,0 +1,6 @@ +# {{ ansible_managed }} + +# Ask crest for the password +password = "$2$au1ywidxo66uz6rnhtpn64f3mn63ykrq$fmgzgxrybouknkaw5or74ho5bwa5ixasiwo7k16p19asgrqs49ay"; +enable_password = "$2$1roytuwoajssbqkdq97ec6nfa3gn6sas$zm5sqsodfmth3cf87dg4pfmmyadhgdsw9dekemfnmwphu65k5p6b"; +bind_socket = "/var/run/haproxy/rspamd/controller mode=0666"; diff --git a/roles/rspamd/templates/override.d/worker-fuzzy.inc.j2 b/roles/rspamd/templates/override.d/worker-fuzzy.inc.j2 new file mode 100644 index 0000000..6d187b6 --- /dev/null +++ b/roles/rspamd/templates/override.d/worker-fuzzy.inc.j2 @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +backend = "redis"; +servers = "/var/run/redis-fuzzy/sock"; diff --git a/roles/rspamd/templates/override.d/worker-normal.inc.j2 b/roles/rspamd/templates/override.d/worker-normal.inc.j2 new file mode 100644 index 0000000..d1fd550 --- /dev/null +++ b/roles/rspamd/templates/override.d/worker-normal.inc.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} + +bind_socket = "/var/run/rspamd/normal.sock"; diff --git a/roles/rspamd/templates/override.d/worker-proxy.inc.j2 b/roles/rspamd/templates/override.d/worker-proxy.inc.j2 new file mode 100644 index 0000000..2bc5281 --- /dev/null +++ b/roles/rspamd/templates/override.d/worker-proxy.inc.j2 @@ -0,0 +1,10 @@ +# {{ ansible_managed }} + +bind_socket = "/var/run/rspamd/proxy.sock mode=0666"; +milter = yes; +timeout = 120s; + +upstream "local" { + default = yes; + self_scan = yes; +} diff --git a/roles/rspamd/templates/rspamd-log/finish.j2 b/roles/rspamd/templates/rspamd-log/finish.j2 new file mode 100644 index 0000000..37c3bce --- /dev/null +++ b/roles/rspamd/templates/rspamd-log/finish.j2 @@ -0,0 +1,13 @@ +#!/usr/local/bin/execlineb -S2 +# {{ ansible_managed }} + +s6-envdir ./env +multisubstitute { + importas -i -u NAME NAME +} + +fdmove -c 1 2 +ifelse { test "${1}" -eq 0 } { + echo "${NAME}-log: Stopped." +} + echo "${NAME}-log: Failed with exit status (${1}, ${2})." diff --git a/roles/rspamd/templates/rspamd-log/run.j2 b/roles/rspamd/templates/rspamd-log/run.j2 new file mode 100644 index 0000000..1f9904f --- /dev/null +++ b/roles/rspamd/templates/rspamd-log/run.j2 @@ -0,0 +1,23 @@ +#!/usr/local/bin/execlineb +# {{ ansible_managed }} + +s6-envdir ./env +multisubstitute { + importas -i -u NAME NAME + importas -i -u USER USER + importas -i -u GROUP GROUP + importas -i -u MODE MODE + importas -i -u DIR DIR +} + +foreground { fdmove -c 1 2 echo "${NAME} log: Starting." } + +ifelse -n { install -d -o "${USER}" -g "${GROUP}" -m "${MODE}" "$DIR" } { + foreground { fdmove -c 1 2 echo "${NAME} log: Failed to create logging directory." } + false +} + +fdmove -c 2 1 + +s6-envuidgid $USER +s6-log -d 3 T $DIR diff --git a/roles/rspamd/templates/rspamd.cfg.j2 b/roles/rspamd/templates/rspamd.cfg.j2 new file mode 100644 index 0000000..3888904 --- /dev/null +++ b/roles/rspamd/templates/rspamd.cfg.j2 @@ -0,0 +1,23 @@ +# {{ ansible_managed }} + +frontend front_rspamd + log global + bind /var/run/haproxy/rspamd.https mode 600 user haproxy group haproxy ssl crt /usr/local/etc/haproxy/{{ ansible_fqdn }}.pem alpn h2,http/1.1 accept-proxy + http-request set-src src,ipmask(16,56) + use_backend back_rspamd + +backend back_rspamd + server rspamd_controller /rspamd/controller + option forwardfor + http-request add-header X-Forwarded-Proto https + http-request add-header X-Forwarded-Port 443 + +backend sni_rspamd + mode tcp + acl clienthello req_ssl_hello_type 1 + acl serverhello rep_ssl_hello_type 2 + tcp-request inspect-delay 5s + tcp-request content accept if clienthello + tcp-response content accept if serverhello + option ssl-hello-chk + server rspamd_https /rspamd.https send-proxy diff --git a/roles/rspamd/templates/rspamd/data/check.j2 b/roles/rspamd/templates/rspamd/data/check.j2 new file mode 100644 index 0000000..15797ce --- /dev/null +++ b/roles/rspamd/templates/rspamd/data/check.j2 @@ -0,0 +1,19 @@ +#!/usr/local/bin/execlineb -P +# {{ ansible_managed }} + +s6-envdir ./env +multisubstitute { + importas -i -u NAME NAME + importas -i -u SOCK SOCK +} + +ifelse { + redirfd -w 1 /dev/null + fdmove -c 2 1 + pipeline { sockstat -l -j 0 -u } fgrep -q "${SOCK}" +} { + foreground { fdmove -c 1 2 echo "${NAME}: Ready." } + true +} + foreground { fdmove -c 1 2 echo "${NAME}: Poll." } + false diff --git a/roles/rspamd/templates/rspamd/finish.j2 b/roles/rspamd/templates/rspamd/finish.j2 new file mode 100644 index 0000000..00e3945 --- /dev/null +++ b/roles/rspamd/templates/rspamd/finish.j2 @@ -0,0 +1,13 @@ +#!/usr/local/bin/execlineb -S2 +# {{ ansible_managed }} + +s6-envdir ./env +multisubstitute { + importas -i -u NAME NAME +} + +fdmove -c 1 2 +ifelse { test "${1}" -eq 0 } { + echo "${NAME}: Stopped." +} + echo "${NAME}: Failed with exit status (${1}, ${2})." diff --git a/roles/rspamd/templates/rspamd/run.j2 b/roles/rspamd/templates/rspamd/run.j2 new file mode 100644 index 0000000..267bcea --- /dev/null +++ b/roles/rspamd/templates/rspamd/run.j2 @@ -0,0 +1,13 @@ +#!/usr/local/bin/execlineb -P +# {{ ansible_managed }} + +s6-envdir ./env +multisubstitute { + importas -i -u NAME NAME +} + +foreground { fdmove -c 1 2 echo "${NAME}: Starting." } +s6-notifyoncheck -d -w 100 -n 70 + +fdmove -c 2 1 +rspamd -f -u rspamd -g rspamd diff --git a/roles/rspamd/vars/main.yml b/roles/rspamd/vars/main.yml new file mode 100644 index 0000000..13f5a5b --- /dev/null +++ b/roles/rspamd/vars/main.yml @@ -0,0 +1,71 @@ +--- +rspamd_log_size: '32m' +rspamd_log_mode: '750' +rspamd_log_uid: '20000' +rspamd_log_gid: '20000' + +rspamd_service_dirs: + - rspamd + - rspamd/env + - rspamd/data + - rspamd-log + - rspamd-log/env + +rspamd_service_scripts: + - rspamd/run + - rspamd/finish + - rspamd/data/check + - rspamd-log/run + - rspamd-log/finish + +rspamd_service_config: + - name: rspamd/type + content: longrun + - name: rspamd/producer-for + content: rspamd-log + - name: rspamd/notification-fd + content: 3 + - name: rspamd/env/NAME + content: rspamd + - name: rspamd/env/PATH + content: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin + - name: rspamd/env/SOCK + content: /var/run/rspamd/normal.sock + - name: rspamd/dependencies + content: | + unbound + redis-bayes + redis-fuzzy + redis-other + + - name: rspamd-log/type + content: longrun + - name: rspamd-log/notification-fd + content: 3 + - name: rspamd-log/consumer-for + content: rspamd + - name: rspamd-log/env/NAME + content: rspamd + - name: rspamd-log/env/PATH + content: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin + - name: rspamd-log/env/MODE + content: '750' + - name: rspamd-log/env/USER + content: s6-log + - name: rspamd-log/env/GROUP + content: s6-log + - name: rspamd-log/env/DIR + content: /var/log/rspamd + +rspamd_config: + - override.d/logging.inc + - override.d/worker-normal.inc + - override.d/worker-controller.inc + - override.d/worker-fuzzy.inc + - override.d/worker-proxy.inc + + - local.d/redis.conf + - local.d/options.inc + - local.d/classifier-bayes.conf + - local.d/composites.conf + - local.d/fann_redis.conf