diff --git a/ansible/inventory/host_files/list-vagrant/fail2ban/jail.local b/ansible/inventory/host_files/list-vagrant/fail2ban/jail.local new file mode 100644 index 0000000..2c47637 --- /dev/null +++ b/ansible/inventory/host_files/list-vagrant/fail2ban/jail.local @@ -0,0 +1,8 @@ +[DEFAULT] +bantime = 3600 + +[sshd] +enabled = true +action = iptables[name=SSH, port=ssh, protocol=tcp] + f2b-warden[name=SSH, port=ssh, protocol=tcp, wardenpath=/data/warden/] + diff --git a/ansible/inventory/host_files/list-vagrant/ipfixcol/profiles.xml b/ansible/inventory/host_files/list-vagrant/ipfixcol/profiles.xml index a01db6c..4b8db73 100644 --- a/ansible/inventory/host_files/list-vagrant/ipfixcol/profiles.xml +++ b/ansible/inventory/host_files/list-vagrant/ipfixcol/profiles.xml @@ -1,75 +1,132 @@ - - normal - - /data/flow/live/ + + normal + + /data/flow/live/ + + + + + + + * + + + ipVersion = 4 + + + + * + + ipv6 + - - - - - - - * - - - ipVersion = 4 - - - - * - - ipv6 - - + + - - - - - normal - /data/flow/live/emails/ + + + + + + normal + /data/flow/live/emails/ + + + + + + ipv4 + ipv6 + + port in [110 995] + + + + + ipv4 + ipv6 + + port in [143 993] + + + + + ipv4 + ipv6 + + port in [25 465] + + + - - - - - - ipv4 - ipv6 - - - port in [110 995 ] - - + + - - - - ipv4 - ipv6 - - - port in [143 993] - - - - - - - ipv4 - ipv6 - - - sourceTransportPort == 25 or - destinationTransportPort == 25 or - sourceTransportPort == 465 or - destinationTransportPort == 465 - - - - - + + diff --git a/ansible/inventory/host_files/list-vagrant/nemea/detectors/blacklistfilter.sup b/ansible/inventory/host_files/list-vagrant/nemea/detectors/blacklistfilter.sup index 65a904c..55c448e 100644 --- a/ansible/inventory/host_files/list-vagrant/nemea/detectors/blacklistfilter.sup +++ b/ansible/inventory/host_files/list-vagrant/nemea/detectors/blacklistfilter.sup @@ -10,7 +10,7 @@ ipblacklistfilter true /usr/bin/nemea/ipblacklistfilter - + UNIXSOCKET diff --git a/ansible/inventory/host_files/list-vagrant/nemea/detectors/blacklistfilter_with_af.sup.example b/ansible/inventory/host_files/list-vagrant/nemea/detectors/blacklistfilter_with_af.sup.example new file mode 100644 index 0000000..18e67f1 --- /dev/null +++ b/ansible/inventory/host_files/list-vagrant/nemea/detectors/blacklistfilter_with_af.sup.example @@ -0,0 +1,158 @@ + +blacklist_downloader +true +/usr/bin/nemea/bl_downloader.py +--repo-path /data/blacklistfilter/blacklist_repo + + + + +ipblacklistfilter +true +/usr/bin/nemea/ipblacklistfilter + + + + UNIXSOCKET + IN + flow_data_source + + + UNIXSOCKET + OUT + ipblacklist_aggregator_ur_sock + + + + + +ipblacklist_aggregator_ur +true +/usr/bin/nemea/agg +-k SRC_IP -k DST_IP -k PROTOCOL -k DST_PORT -s BYTES -s PACKETS -o SRC_BLACKLIST -o DST_BLACKLIST -t g:60 + + + UNIXSOCKET + IN + ipblacklist_aggregator_ur_sock + + + UNIXSOCKET + OUT + blacklist_aggregator_ip:timeout=1000 + + + + + +blacklist_aggregator_json +true +/usr/bin/nemea/blacklist_aggregator.py +-t 5 + + + UNIXSOCKET + IN + blacklist_aggregator_ip + + + UNIXSOCKET + IN + blacklist_aggregator_url + + + UNIXSOCKET + OUT + adaptive_ip_url2:timeout=NO_WAIT + + + + + +urlblacklistfilter +true +/usr/bin/nemea/urlblacklistfilter + + + + UNIXSOCKET + IN + http_data_source + + + UNIXSOCKET + OUT + blacklist_aggregator_url + + + + + +dnsblacklistfilter +true +/usr/bin/nemea/dnsblacklistfilter + + + + UNIXSOCKET + IN + dns_data_source + + + UNIXSOCKET + OUT + adaptive_dns + + + + + +adaptive_ipblacklistfilter +true +/usr/bin/nemea/ipblacklistfilter +-4 /tmp/blacklistfilter/adaptive.blist + + + UNIXSOCKET + IN + flow_data_source + + + FILE + OUT + /data/blacklistfilter/evidence_adaptive + + + + + +adaptive_filter +true +/usr/bin/nemea/adaptive_filter +-p 30 -l 20 -e 300 + + + UNIXSOCKET + IN + adaptive_ip_url2 + + + + UNIXSOCKET + IN + adaptive_dns + + + + UNIXSOCKET + OUT + blacklist2idea_sock:timeout=NO_WAIT + + + + FILE + OUT + /data/blacklistfilter/evidence_detection + + + \ No newline at end of file diff --git a/ansible/inventory/host_files/list-vagrant/nemea/detectors/dnstunnel_detection.sup b/ansible/inventory/host_files/list-vagrant/nemea/detectors/dnstunnel_detection.sup index c7e701e..2ab9d6c 100644 --- a/ansible/inventory/host_files/list-vagrant/nemea/detectors/dnstunnel_detection.sup +++ b/ansible/inventory/host_files/list-vagrant/nemea/detectors/dnstunnel_detection.sup @@ -6,7 +6,7 @@ UNIXSOCKET IN - flow_data_source + dns_data_source TCP diff --git a/ansible/inventory/host_files/list-vagrant/nemea/loggers/blacklistfilter_logger.sup b/ansible/inventory/host_files/list-vagrant/nemea/loggers/blacklistfilter_logger.sup deleted file mode 100644 index c32c3aa..0000000 --- a/ansible/inventory/host_files/list-vagrant/nemea/loggers/blacklistfilter_logger.sup +++ /dev/null @@ -1,13 +0,0 @@ - -blacklistfilter_aggr_logger -true -/usr/bin/nemea/json_dump.py --a /data/blacklistfilter/detected.log - - - TCP - IN - 12006 - - - diff --git a/ansible/inventory/host_files/list-vagrant/nemea/reporters-config.yml b/ansible/inventory/host_files/list-vagrant/nemea/reporters-config.yml index 2ad42c3..588d1eb 100644 --- a/ansible/inventory/host_files/list-vagrant/nemea/reporters-config.yml +++ b/ansible/inventory/host_files/list-vagrant/nemea/reporters-config.yml @@ -1,3 +1,5 @@ +--- +namespace: com.example.list_vagrant.nemea smtp_connections: - id: mylocalserver custom_actions: @@ -6,9 +8,13 @@ custom_actions: host: localhost db: nemeadb collection: alerts_new - - id: store_warden + - id: warden warden: url: https://localhost/warden/ + - id: store_warden + file: + path: /data/warden/incoming/ + temp_path: /data/warden/temp/ - id: file file: path: /dev/stdout diff --git a/ansible/inventory/host_files/list-vagrant/nemea/reporters/reporters.sup b/ansible/inventory/host_files/list-vagrant/nemea/reporters/reporters.sup index 4f3faa8..d155ee3 100644 --- a/ansible/inventory/host_files/list-vagrant/nemea/reporters/reporters.sup +++ b/ansible/inventory/host_files/list-vagrant/nemea/reporters/reporters.sup @@ -1,156 +1,160 @@ - hoststats2idea - true - /usr/bin/nemea/hoststats2idea.py - -n cz.cesnet.nemea.hoststats --warden=/etc/warden/hoststats.cfg -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12002 - - + hoststats2idea + true + /usr/bin/python3 + /usr/bin/nemea/hoststats2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12002 + + - amplification2idea - true - /usr/bin/nemea/amplification2idea.py - -n cz.cesnet.nemea.amplificationdetector --warden=/etc/warden/amplificationdetector.cfg -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12001 - - + amplification2idea + true + /usr/bin/python3 + /usr/bin/nemea/amplification2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12001 + + - blacklist2idea - true - /usr/bin/nemea/blacklist2idea.py - --name=cz.cesnet.nemea.blacklist --warden=/etc/warden/blacklist.cfg -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12006 - - + blacklist2idea + true + /usr/bin/python3 + /usr/bin/nemea/blacklist2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12006 + + - vportscan2idea - true - /usr/bin/nemea/vportscan2idea.py - -n cz.cesnet.nemea.vportscan --warden=/etc/warden/vportscan.cfg -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12005 - - + vportscan2idea + true + /usr/bin/python3 + /usr/bin/nemea/vportscan2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12005 + + - bruteforce2idea - true - /usr/bin/nemea/bruteforce2idea.py - -n cz.cesnet.nemea.bruteforce --warden=/etc/warden/bruteforce.cfg -c /etc/nemea/reporters-config.yml - - - UNIXSOCKET - IN - bfd_data_out - - + bruteforce2idea + true + /usr/bin/python3 + /usr/bin/nemea/bruteforce2idea.py -c /etc/nemea/reporters-config.yml + + + UNIXSOCKET + IN + bfd_data_out + + - voipfraud2idea - true - /usr/bin/nemea/voipfraud2idea.py - -n cz.cesnet.nemea.voipfraud_detection --warden=/etc/warden/voipfraud_detection.cfg -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12003 - - + voipfraud2idea + true + /usr/bin/python3 + /usr/bin/nemea/voipfraud2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12003 + + - dnstunnel2idea - false - /usr/bin/nemea/dnstunnel2idea.py - -n cz.cesnet.nemea.dnstunnel --warden=/etc/warden/dnstunnel.cfg -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12004 - - + dnstunnel2idea + false + /usr/bin/python3 + /usr/bin/nemea/dnstunnel2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12004 + + - - -haddrscan2idea -true -/usr/bin/nemea/haddrscan2idea.py --n cz.cesnet.nemea.haddrscan -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12008 - - + haddrscan2idea + true + /usr/bin/python3 + /usr/bin/nemea/haddrscan2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12008 + + - ddos_detector2idea - true - /usr/bin/nemea/ddos_detector2idea.py - -n cz.cesnet.nemea.ddos_detector -c /etc/nemea/reporters-config.yml - - - UNIXSOCKET - IN - ddos_detector_alert - - + ddos_detector2idea + true + /usr/bin/python3 + /usr/bin/nemea/ddos_detector2idea.py -c /etc/nemea/reporters-config.yml + + + UNIXSOCKET + IN + ddos_detector_alert + + -minerdetector2idea -false -/usr/bin/nemea/minerdetector2idea.py --n cz.cesnet.nemea.miner_detector -c /etc/nemea/reporters-config.yml - - - UNIXSOCKET - IN - miner_detector_data_out - - + minerdetector2idea + false + /usr/bin/python3 + /usr/bin/nemea/minerdetector2idea.py -c /etc/nemea/reporters-config.yml + + + UNIXSOCKET + IN + miner_detector_data_out + + -sipbf2idea -true -/usr/bin/nemea/sipbf2idea.py --n cz.cesnet.nemea.sip_bf_detector -c /etc/nemea/reporters-config.yml - - - TCP - IN - 12009 - - + sipbf2idea + true + /usr/bin/python3 + /usr/bin/nemea/sipbf2idea.py -c /etc/nemea/reporters-config.yml + + + TCP + IN + 12009 + + + + warden_filer + true + /usr/bin/nemea/nemea_warden_filer + -c /etc/nemea/reporters-config.yml -w /etc/warden/filer/warden_client.cfg + diff --git a/ansible/inventory/host_files/list-vagrant/nemea/supervisor_config_template.xml b/ansible/inventory/host_files/list-vagrant/nemea/supervisor_config_template.xml index b4f510a..86076d0 100644 --- a/ansible/inventory/host_files/list-vagrant/nemea/supervisor_config_template.xml +++ b/ansible/inventory/host_files/list-vagrant/nemea/supervisor_config_template.xml @@ -21,8 +21,9 @@ - Loggers + Munin true - + + diff --git a/ansible/inventory/host_files/list-vagrant/socat/tls-endpoint.service b/ansible/inventory/host_files/list-vagrant/socat/tls-endpoint.service new file mode 100644 index 0000000..88b4db0 --- /dev/null +++ b/ansible/inventory/host_files/list-vagrant/socat/tls-endpoint.service @@ -0,0 +1,21 @@ +[Unit] +Description=Create a simple TLS endpoint using socat and keys&certs in /etc/tls-enspoint/ (server.key,server.crt,sa.crt) +After=network-online.target +Before=multi-user.target +DefaultDependencies=no + +[Service] +User=tlsendpoint + +ExecStart=/bin/socat openssl-listen:4740,method=TLS1.2,key=/etc/tls-endpoint/server.key,cert=/etc/tls-endpoint/server.crt,cafile=/etc/tls-endpoint/ca.crt,reuseaddr,fork udp-sendto:localhost:4739 + +# wait 60 seconds before trying to restart the connection +# if it disconnects +RestartSec=60 + +# keep retrying no matter what +Restart=always + +[Install] +WantedBy=multi-user.target + diff --git a/ansible/list.yml b/ansible/list.yml index ec54797..0cf7953 100644 --- a/ansible/list.yml +++ b/ansible/list.yml @@ -1,5 +1,7 @@ - hosts: list force_handlers: true + environment: + PATH: /usr/local/bin:{{ ansible_env.PATH }} roles: - { role: settings, tags: settings } - { role: nemea-dashboard, tags: nemea-dashboard } @@ -10,4 +12,7 @@ - { role: ipfixcol, tags: ipfixcol } - { role: nemea-status, tags: nemea-status } - { role: main-page, tags: main-page } - - { role: warden-client, tags: warden-client } \ No newline at end of file + - { role: warden-client, tags: warden-client } + - { role: easyrsa, tags: easyrsa } + - { role: fail2ban, tags: fail2ban } + - { role: socat, tags: socat } diff --git a/ansible/roles/apache/tasks/install.yml b/ansible/roles/apache/tasks/install.yml index 4bfb4d9..28cdb09 100644 --- a/ansible/roles/apache/tasks/install.yml +++ b/ansible/roles/apache/tasks/install.yml @@ -1,6 +1,22 @@ - name: Install Apache yum: "name={{ packages }} state=installed" +- name: Create /etc/systemd/system/httpd.service.d + file: + path: /etc/systemd/system/httpd.service.d + state: directory + +- name: Install systemd privatetmp override + copy: + content: < + [Service] + PrivateTmp=False + dest: /etc/systemd/system/httpd.service.d/noprivtemp.conf + +- name: Reload systemd + systemd: + daemon_reload: yes + - name: Redirect HTTP to HTTPs copy: src: "{{ apache_redirect_conf.src }}" diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index b83d1d2..ed46495 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -20,4 +20,20 @@ set_fact: ansible_ssh_private_key_file: "{{ ansible_ssh_private_key_file | realpath }}" when: ansible_ssh_private_key_file is defined - tags: always \ No newline at end of file + tags: always + +- name: Create nemead group + group: + name: nemead + state: present + tags: install + +- name: Create nemead user + user: + name: nemead + comment: NEMEA system and related tools + group: nemead + system: True + state: present + tags: install + diff --git a/ansible/roles/easyrsa/tasks/install.yml b/ansible/roles/easyrsa/tasks/install.yml new file mode 100644 index 0000000..021e829 --- /dev/null +++ b/ansible/roles/easyrsa/tasks/install.yml @@ -0,0 +1,13 @@ +- name: Download EasyRSA archive + get_url: + url: "{{ easyrsa_src.url }}" + dest: "{{ easyrsa_src.dest }}" + +- name: Extract EasyRSA + shell: | + mkdir -p "{{ easyrsa_src.creates }}" + tar -C "{{ easyrsa_src.creates }}" --strip-components=1 -xzf "{{ easyrsa_src.dest }}" + rm "{{ easyrsa_src.dest }}" + args: + creates: "{{ easyrsa_src.creates }}" + diff --git a/ansible/roles/easyrsa/tasks/main.yml b/ansible/roles/easyrsa/tasks/main.yml new file mode 100644 index 0000000..ab5be72 --- /dev/null +++ b/ansible/roles/easyrsa/tasks/main.yml @@ -0,0 +1,4 @@ +- name: EasyRSA install + include: install.yml + tags: install + diff --git a/ansible/roles/easyrsa/vars/main.yml b/ansible/roles/easyrsa/vars/main.yml new file mode 100644 index 0000000..48dcb2e --- /dev/null +++ b/ansible/roles/easyrsa/vars/main.yml @@ -0,0 +1,8 @@ +easyrsa_tmp: "/tmp" + +easyrsa_src: { + url: "https://github.com/OpenVPN/easy-rsa/releases/download/v3.0.6/EasyRSA-unix-v3.0.6.tgz", + dest: "{{ easyrsa_tmp }}/EasyRSA-unix-v3.0.6.tgz", + creates: "/opt/easyrsa" +} + diff --git a/ansible/roles/fail2ban/files/f2b-warden.conf b/ansible/roles/fail2ban/files/f2b-warden.conf new file mode 100644 index 0000000..c3133e9 --- /dev/null +++ b/ansible/roles/fail2ban/files/f2b-warden.conf @@ -0,0 +1,19 @@ +[Definition] + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = /usr/bin/nemea/f2ban_ssh.sh -i "" -f "" -t " - + + + u + http_data_source + 1000 + 1000000 + 1 + DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,?HTTP_REQUEST_AGENT_ID,?HTTP_REQUEST_METHOD_ID,?HTTP_RESPONSE_STATUS_CODE,PACKETS,DST_PORT,SRC_PORT,DIR_BIT_FIELD,PROTOCOL,TCP_FLAGS,?TOS,?TTL,?HTTP_REQUEST_AGENT,?HTTP_REQUEST_HOST,?HTTP_REQUEST_REFERER,?HTTP_REQUEST_URL,?HTTP_RESPONSE_CONTENT_TYPE + + + + u + dns_data_source + 1000 + 1000000 + 1 + DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,DNS_RR_TTL,PACKETS,DNS_ANSWERS,DNS_CLASS,DNS_ID,?DNS_PSIZE,DNS_QTYPE,DNS_RLENGTH,?DST_PORT,?SRC_PORT,DIR_BIT_FIELD,?DNS_DO,DNS_RCODE,PROTOCOL,?TCP_FLAGS,?TOS,?TTL,?DNS_NAME,?DNS_RDATA + + u voip_data_source @@ -111,4 +149,3 @@ src: "/usr/bin/nemea/link_traff2json.py" mode: 0755 remote_src: yes - diff --git a/ansible/roles/nemea/vars/main.yml b/ansible/roles/nemea/vars/main.yml index 253524e..9dd9c41 100644 --- a/ansible/roles/nemea/vars/main.yml +++ b/ansible/roles/nemea/vars/main.yml @@ -1,4 +1,4 @@ -packages: [nemea, nemea-modules] +packages: [nemea, python36-nemea-pycommon, python36-nemea-pytrap] repo: { nemea: { diff --git a/ansible/roles/php7/tasks/install.yml b/ansible/roles/php7/tasks/install.yml new file mode 100644 index 0000000..b8c93f8 --- /dev/null +++ b/ansible/roles/php7/tasks/install.yml @@ -0,0 +1,20 @@ +- name: Install remirepo with PHP packages + yum: + name: "{{ repourl }}" + state: present + +- name: Install yum-utils + yum: + name: "{{ phprepoconfiger }}" + state: installed + +- name: Select php73 + shell: yum-config-manager --enable remi-php73 + +- name: Install PHP packages + yum: + name: "{{ phppackages }}" + state: installed + notify: + - Apache restart + diff --git a/ansible/roles/pip3.4/tasks/main.yml b/ansible/roles/php7/tasks/main.yml similarity index 100% rename from ansible/roles/pip3.4/tasks/main.yml rename to ansible/roles/php7/tasks/main.yml diff --git a/ansible/roles/php7/vars/main.yml b/ansible/roles/php7/vars/main.yml new file mode 100644 index 0000000..dcceafc --- /dev/null +++ b/ansible/roles/php7/vars/main.yml @@ -0,0 +1,4 @@ +repourl: http://rpms.remirepo.net/enterprise/remi-release-7.rpm +phprepoconfiger: [yum-utils] +phppackages: [php, php-xml] + diff --git a/ansible/roles/pip3.4/tasks/install.yml b/ansible/roles/pip3.4/tasks/install.yml deleted file mode 100644 index d508bcd..0000000 --- a/ansible/roles/pip3.4/tasks/install.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Install dependencies - yum: "name={{ packages }} state=installed" - -- name: Get python pip - get_url: - url: "{{ pip.url }}" - dest: "{{ pip.tmp }}" - -- name: Install python pip - command: python3.4 "{{ pip.tmp }}" creates="{{ pip.creates }}" \ No newline at end of file diff --git a/ansible/roles/pip3.4/vars/main.yml b/ansible/roles/pip3.4/vars/main.yml deleted file mode 100644 index 229d753..0000000 --- a/ansible/roles/pip3.4/vars/main.yml +++ /dev/null @@ -1,7 +0,0 @@ -packages: [python34, python34-devel, "@Development tools"] - -pip: { - url: "https://bootstrap.pypa.io/get-pip.py", - tmp: "/tmp/get-pip.py", - creates: "/bin/pip", -} \ No newline at end of file diff --git a/ansible/roles/pip3.4/meta/main.yml b/ansible/roles/python3/meta/main.yml similarity index 100% rename from ansible/roles/pip3.4/meta/main.yml rename to ansible/roles/python3/meta/main.yml diff --git a/ansible/roles/python3/tasks/install.yml b/ansible/roles/python3/tasks/install.yml new file mode 100644 index 0000000..09f5b75 --- /dev/null +++ b/ansible/roles/python3/tasks/install.yml @@ -0,0 +1,10 @@ +- name: Install dependencies + yum: "name={{ packages }} state=installed" + +- name: Update pip3 + pip: + name: + - setuptools + executable: pip3 + state: latest + diff --git a/ansible/roles/python3/tasks/main.yml b/ansible/roles/python3/tasks/main.yml new file mode 100644 index 0000000..3fe54f9 --- /dev/null +++ b/ansible/roles/python3/tasks/main.yml @@ -0,0 +1,2 @@ +- include: install.yml + tags: install \ No newline at end of file diff --git a/ansible/roles/python3/vars/main.yml b/ansible/roles/python3/vars/main.yml new file mode 100644 index 0000000..01f13cd --- /dev/null +++ b/ansible/roles/python3/vars/main.yml @@ -0,0 +1,2 @@ +packages: [python36, python36-devel, python36-pip, python36-virtualenv, "@Development tools"] + diff --git a/ansible/roles/scgui/meta/main.yml b/ansible/roles/scgui/meta/main.yml index 1dc5100..b2d887e 100644 --- a/ansible/roles/scgui/meta/main.yml +++ b/ansible/roles/scgui/meta/main.yml @@ -1,5 +1,6 @@ dependencies: - role: common - role: apache + - role: php7 - { role: ipfixcol, tags: ipfixcol } - { role: nemea, tags: nemea } diff --git a/ansible/roles/socat/tasks/install.yml b/ansible/roles/socat/tasks/install.yml new file mode 100644 index 0000000..fc4a854 --- /dev/null +++ b/ansible/roles/socat/tasks/install.yml @@ -0,0 +1,40 @@ +- name: Check for TLS endpoint configuration (tls-endpoint.service) + local_action: stat path={{ tlsendpoint_service }} + become: false + register: tlsendpoint_service_file + ignore_errors: True + +- name: Install socat + yum: "name=socat state=installed" + when: tlsendpoint_service_file.stat.exists + +- name: Copy tls-endpoint.service configuration + copy: + src: "{{ tlsendpoint_service }}" + dest: /etc/systemd/system/tls-endpoint.service + when: tlsendpoint_service_file.stat.exists + +- name: Create a system user tlsendpoint + user: + name: tlsendpoint + system: yes + state: present + create_home: no + when: tlsendpoint_service_file.stat.exists + +- name: Create directory for TLS certificates + file: + path: /etc/tls-endpoint/ + state: directory + owner: tlsendpoint + when: tlsendpoint_service_file.stat.exists + +- name: Create README for tlsendpoint + copy: + content: > + Put server.key, server.crt, ca.crt into this directory and execute + systemctl enable tlsendpoint + service tlsendpoint start + dest: /etc/tls-endpoint/README + when: tlsendpoint_service_file.stat.exists + diff --git a/ansible/roles/socat/tasks/main.yml b/ansible/roles/socat/tasks/main.yml new file mode 100644 index 0000000..97d6935 --- /dev/null +++ b/ansible/roles/socat/tasks/main.yml @@ -0,0 +1,4 @@ +- name: Socat install + include: install.yml + tags: install + diff --git a/ansible/roles/socat/vars/main.yml b/ansible/roles/socat/vars/main.yml new file mode 100644 index 0000000..bdcb242 --- /dev/null +++ b/ansible/roles/socat/vars/main.yml @@ -0,0 +1,2 @@ +tlsendpoint_service: "{{ inventory_dir }}/host_files/{{ inventory_hostname }}/socat/tls-endpoint.service" + diff --git a/ansible/roles/warden-client/files/warden_client.cfg b/ansible/roles/warden-client/files/warden_client.cfg new file mode 100644 index 0000000..f113885 --- /dev/null +++ b/ansible/roles/warden-client/files/warden_client.cfg @@ -0,0 +1,20 @@ +{ + "warden": { + "url": "https://warden-hub.cesnet.cz/warden3", + "certfile": "/etc/warden/filer/cert.pem", + "keyfile": "/etc/warden/filer/key.pem", + "timeout": 60, + "get_events_limit": 1000, + "syslog": {"level": "warning"}, + "idstore": "warden_client.id", + "name": "automatically-generated" + }, + "sender": { + "dir": "/data/warden/", + "node": { + "Name": "automatically-generated", + "Type": ["Relay"] + } + } +} + diff --git a/ansible/roles/warden-client/files/warden_client.py b/ansible/roles/warden-client/files/warden_client.py index 3604e56..6f4b3f1 100644 --- a/ansible/roles/warden-client/files/warden_client.py +++ b/ansible/roles/warden-client/files/warden_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright (C) 2011-2015 Cesnet z.s.p.o diff --git a/ansible/roles/warden-client/files/warden_filer.py b/ansible/roles/warden-client/files/warden_filer.py index c7ef609..2c410d9 100644 --- a/ansible/roles/warden-client/files/warden_filer.py +++ b/ansible/roles/warden-client/files/warden_filer.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright (C) 2011-2015 Cesnet z.s.p.o @@ -428,7 +428,7 @@ def get_configs(): # Allow inline or external Warden config wconfig = config.get("warden", "warden_client.cfg") - if isinstance(wconfig, basestring): + if isinstance(wconfig, str): wconfig = read_cfg(wconfig) fconfig = config.get(args.func, {}) diff --git a/ansible/roles/warden-client/tasks/common.yml b/ansible/roles/warden-client/tasks/common.yml index 777f2c8..08db4c2 100644 --- a/ansible/roles/warden-client/tasks/common.yml +++ b/ansible/roles/warden-client/tasks/common.yml @@ -12,3 +12,88 @@ dest: "{{ warden_filer.dest }}" mode: "{{ warden_filer.mode }}" +- name: Create config dir for warden_filer sender + file: + path: "/etc/warden/filer" + state: directory + mode: '0750' + owner: nemead + group: nemead + +- name: Copy config for warden_filer sender + copy: + src: "{{ warden_filer.configsrc }}" + dest: "/etc/warden/filer/warden_client.cfg" + owner: nemead + group: nemead + mode: "0640" + +- name: Check for generated key and cert + stat: path=/opt/warden_server_3/keys/client.key + register: warden_filer_key + ignore_errors: True + +- name: Copy warden_filer key + copy: + remote_src: True + src: "/opt/warden_server_3/keys/client.key" + dest: "/etc/warden/filer/key.pem" + mode: "0640" + owner: nemead + group: nemead + when: warden_filer_key.stat.exists + +- name: Copy warden_filer ca cert + copy: + remote_src: True + src: "/opt/warden_server_3/ca/rootCA.pem" + dest: "/etc/warden/filer/rootCA.pem" + mode: "0640" + owner: nemead + group: nemead + when: warden_filer_key.stat.exists + register: warden_ca_cert + ignore_errors: True + +- name: Copy warden_filer cert + copy: + remote_src: True + src: "/opt/warden_server_3/keys/client.crt" + dest: "/etc/warden/filer/cert.pem" + mode: "0640" + owner: nemead + group: nemead + when: warden_filer_key.stat.exists + +- name: Insert path to CA cert into config + blockinfile: + dest: "/etc/warden/filer/warden_client.cfg" + insertbefore: '"keyfile": "/etc/warden/filer/key.pem",' + marker: '"ansiblecafile{mark}": "1",' + backup: yes + block: '"cafile": "/etc/warden/filer/rootCA.pem",' + when: warden_ca_cert is succeeded + +- name: Replace Warden server with localhost in config + replace: + path: "/etc/warden/filer/warden_client.cfg" + regexp: "warden-hub.cesnet.cz" + replace: 'localhost:8443' + when: warden_ca_cert is succeeded + +- name: Clean markers + replace: + path: "/etc/warden/filer/warden_client.cfg" + regexp: '.*ansiblecafile.*' + replace: '' + when: warden_ca_cert is succeeded + + +- name: Create warden dir for NEMEA + file: + path: /data/warden/ + state: directory + owner: nemead + group: nemead + + diff --git a/ansible/roles/warden-client/vars/main.yml b/ansible/roles/warden-client/vars/main.yml index 9fe0b39..970ea2e 100644 --- a/ansible/roles/warden-client/vars/main.yml +++ b/ansible/roles/warden-client/vars/main.yml @@ -1,12 +1,13 @@ warden_filer: { src: "warden_filer.py", + configsrc: "warden_client.cfg", dest: "/usr/local/bin/warden_filer.py", mode: "0755" } warden_client_file: { src: "warden_client.py", - dest: "/usr/lib/python2.7/site-packages/warden_client.py", + dest: "/usr/lib/python3.6/site-packages/warden_client.py", owner: "root", group: "root", mode: "0664" diff --git a/ansible/roles/warden-server/files/httpd/conf.modules.d-warden/10-wsgi.conf b/ansible/roles/warden-server/files/httpd/conf.modules.d-warden/10-wsgi.conf index 7288925..c82a4b9 100644 --- a/ansible/roles/warden-server/files/httpd/conf.modules.d-warden/10-wsgi.conf +++ b/ansible/roles/warden-server/files/httpd/conf.modules.d-warden/10-wsgi.conf @@ -1,2 +1,3 @@ -LoadModule wsgi_module modules/mod_wsgi.so +#LoadModule wsgi_module modules/mod_wsgi.so #LoadModule wsgi_module /usr/lib64/python3.4/site-packages/mod_wsgi/server/mod_wsgi-py34.cpython-34m.so +LoadModule wsgi_module /usr/local/lib64/python3.6/site-packages/mod_wsgi/server/mod_wsgi-py36.cpython-36m-x86_64-linux-gnu.so diff --git a/ansible/roles/warden-server/files/install/generate-warden-cfg.sh b/ansible/roles/warden-server/files/install/generate-warden-cfg.sh index 9de07d0..e8e911b 100755 --- a/ansible/roles/warden-server/files/install/generate-warden-cfg.sh +++ b/ansible/roles/warden-server/files/install/generate-warden-cfg.sh @@ -2,18 +2,11 @@ HOSTNAME=`hostname -f` mkdir /etc/warden -for name in hoststats vportscan amplificationdetector blacklist bruteforce voipfraud_detection dnstunnel; do - secret=`tr -dc '[:alnum:]' /etc/warden/$name.cfg <= 3: import configparser as ConfigParser from urllib.parse import parse_qs + unicode = str + + def get_method_params(method): + return method.__code__.co_varnames[:method.__code__.co_argcount] + else: import ConfigParser from urlparse import parse_qs + def get_method_params(method): + return method.func_code.co_varnames[:method.func_code.co_argcount] + + # for local version of up to date jsonschema sys.path.append(path.join(path.dirname(__file__), "..", "lib")) @@ -139,6 +148,7 @@ def get_clean_root_logger(level=logging.INFO): logger = logging.getLogger(__name__) logger.setLevel(level) while logger.handlers: + logger.handlers[0].close() logger.removeHandler(logger.handlers[0]) while logger.filters: logger.removeFilter(logger.filters[0]) @@ -212,7 +222,7 @@ def SysLogger(req, socket="/dev/log", facility=logging.handlers.SysLogHandler.LO class Object(object): def __str__(self): - attrs = self.__init__.func_code.co_varnames[1:self.__init__.func_code.co_argcount] + attrs = get_method_params(self.__init__)[1:] eq_str = ["%s=%r" % (attr, getattr(self, attr, None)) for attr in attrs] return "%s(%s)" % (type(self).__name__, ", ".join(eq_str)) @@ -435,7 +445,7 @@ class JSONSchemaValidator(NoValidator): def __init__(self, req, log, filename=None): NoValidator.__init__(self, req, log) self.path = filename or path.join(path.dirname(__file__), "idea.schema") - with open(self.path) as f: + with io.open(self.path, "r", encoding="utf-8") as f: self.schema = json.load(f) self.validator = Draft4Validator(self.schema) @@ -477,11 +487,11 @@ def __init__( self.catmap_filename = catmap_filename self.tagmap_filename = tagmap_filename - with open(catmap_filename, "r") as catmap_fd: + with io.open(catmap_filename, "r", encoding="utf-8") as catmap_fd: self.catmap = json.load(catmap_fd) self.catmap_other = self.catmap["Other"] # Catch error soon, avoid lookup later - with open(tagmap_filename, "r") as tagmap_fd: + with io.open(tagmap_filename, "r", encoding="utf-8") as tagmap_fd: self.tagmap = json.load(tagmap_fd) self.tagmap_other = self.catmap["Other"] # Catch error soon, avoid lookup later @@ -595,7 +605,7 @@ def get_client_by_name(self, cert_names=None, name=None, secret=None): with attempt as db: rows = db.query("".join(query), params).fetchall() if len(rows) > 1: - self.log.warn( + self.log.warning( "get_client_by_name: query returned more than one result (cert_names = %s, name = %s, secret = %s): %s" % ( cert_names, name, secret, ", ".join([str(Client(**row)) for row in rows]))) return None @@ -806,10 +816,10 @@ def getLastReceivedId(self, client): def load_maps(self): with self as db: db.query("DELETE FROM tags") - for tag, num in self.tagmap.iteritems(): + for tag, num in self.tagmap.items(): db.query("INSERT INTO tags(id, tag) VALUES (%s, %s)", (num, tag)) db.query("DELETE FROM categories") - for cat_subcat, num in self.catmap.iteritems(): + for cat_subcat, num in self.catmap.items(): catsplit = cat_subcat.split(".", 1) category = catsplit[0] subcategory = catsplit[1] if len(catsplit) > 1 else None @@ -912,8 +922,20 @@ def wsgi_app(self, environ, start_response, exc_info=None): args = self.sanitize_args(path, method, args) + # Based on RFC2616, section 4.4 we SHOULD respond with 400 (bad request) or 411 + # (length required) if content length was not specified. We choose not to, to + # preserve compatibility with clients deployed in the wild, which use POST for + # all requests (even those without payload, with no specified content length). + # According to PEP3333, section "Input and Error Streams", the application SHOULD + # NOT attempt to read more data than specified by CONTENT_LENGTH. As stated in + # section "environ Variables", CONTENT_LENGTH may be empty (string) or absent. try: - post_data = environ['wsgi.input'].read() + content_length = int(environ.get('CONTENT_LENGTH', 0)) + except ValueError: + content_length = 0 + + try: + post_data = environ['wsgi.input'].read(content_length) except: raise self.req.error(message="Data read error.", error=408, exc=sys.exc_info()) @@ -931,10 +953,15 @@ def wsgi_app(self, environ, start_response, exc_info=None): # Make sure everything is properly encoded - JSON and various function # may spit out unicode instead of str and it gets propagated up (str - # + unicode = unicode). However, the right thing would be to be unicode - # correct among whole source and always decode on input (json module - # does that for us) and on output here. - if isinstance(status, unicode): + # + unicode = unicode). + # For Python2 the right thing would be to be unicode correct among whole + # source and always decode on input (json module does that for us) and + # on output here. + # For Python3 strings are internally unicode so no decoding on input is + # necessary. For output, "status" must be unicode string, "output" must + # be encoded bytes array, what is done here. Important: for Python 3 we + # define: unicode = str + if isinstance(status, unicode) and sys.version_info[0] < 3: status = status.encode("utf-8") if isinstance(output, unicode): output = output.encode("utf-8") @@ -947,11 +974,10 @@ def wsgi_app(self, environ, start_response, exc_info=None): def json_wrapper(method): - def meth_deco(self, post, **args): - if "events" in method.func_code.co_varnames[0:method.func_code.co_argcount]: + if "events" in get_method_params(method): try: - events = json.loads(post) if post else None + events = json.loads(post.decode('utf-8')) if post else None except Exception as e: raise self.req.error( message="Deserialization error.", error=400, @@ -973,7 +999,7 @@ def meth_deco(self, post, **args): try: meth_deco.arguments = method.arguments except AttributeError: - meth_deco.arguments = method.func_code.co_varnames[:method.func_code.co_argcount] + meth_deco.arguments = get_method_params(method) return meth_deco @@ -997,13 +1023,14 @@ def __init__( def getDebug(self): return { "environment": self.req.env, - "client": self.req.client.__dict__, + "client": self.req.client._asdict(), "database": self.db.get_debug(), "system": { + "python": sys.version, "uname": os.uname() }, "process": { - "cwd": os.getcwdu(), + "cwd": unicode(os.getcwd()), "pid": os.getpid(), "ppid": os.getppid(), "pgrp": os.getpgrp(), @@ -1177,15 +1204,15 @@ def read_ini(path): def read_cfg(path): - with open(path, "r") as f: + with io.open(path, "r", encoding="utf-8") as f: stripcomments = "\n".join((l for l in f if not l.lstrip().startswith(("#", "//")))) conf = json.loads(stripcomments) # Lowercase keys conf = dict(( sect.lower(), dict( - (subkey.lower(), val) for subkey, val in subsect.iteritems()) - ) for sect, subsect in conf.iteritems()) + (subkey.lower(), val) for subkey, val in subsect.items()) + ) for sect, subsect in conf.items()) return conf @@ -1333,7 +1360,7 @@ def obj(name): } def init_obj(sect_name): - config = conf.get(sect_name, {}) + config = dict(conf.get(sect_name, {})) sect_name = sect_name.lower() sect_def = section_def[sect_name] @@ -1360,7 +1387,7 @@ def init_obj(sect_name): # Process parameters kwargs = {} - for name, definition in params.iteritems(): + for name, definition in params.items(): raw_val = config.get(name, definition["default"]) try: type_callable = conv_dict[definition["type"]] @@ -1451,9 +1478,8 @@ def isValidNSID(nsid): return allowed.match(nsid) def isValidEmail(mail): - mails = (email.utils.parseaddr(m) for m in mail.split(",")) - allowed = re.compile(r"^[a-zA-Z0-9_.%!+-]+@[a-zA-Z0-9-.]+$") # just basic check - valid = (allowed.match(ms[1]) for ms in mails) + allowed = re.compile(r"(^[a-zA-Z0-9_ .%!+-]*(?=<.*>))?(^|(<(?=.*(>))))[a-zA-Z0-9_.%!+-]+@[a-zA-Z0-9-.]+\4?$") # just basic check + valid = (allowed.match(ms.strip())for ms in mail.split(',')) return all(valid) def isValidID(id): @@ -1573,7 +1599,8 @@ def get_args(): argp.add_argument( "-c", "--config", help="path to configuration file") - subargp = argp.add_subparsers(title="commands") + subargp = argp.add_subparsers(title="commands", dest="command") + subargp.required = True subargp_check = subargp.add_parser( "check", add_help=False, diff --git a/ansible/roles/warden-server/files/warden3/warden_server_3/warden_server.wsgi b/ansible/roles/warden-server/files/warden3/warden_server_3/warden_server.wsgi index 609ab80..c141acc 100755 --- a/ansible/roles/warden-server/files/warden3/warden_server_3/warden_server.wsgi +++ b/ansible/roles/warden-server/files/warden3/warden_server_3/warden_server.wsgi @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright (C) 2011-2013 Cesnet z.s.p.o diff --git a/ansible/roles/warden-server/meta/main.yml b/ansible/roles/warden-server/meta/main.yml index f24e291..43e5e0e 100644 --- a/ansible/roles/warden-server/meta/main.yml +++ b/ansible/roles/warden-server/meta/main.yml @@ -1,4 +1,5 @@ dependencies: - common - epel + - python3 - apache diff --git a/ansible/roles/warden-server/tasks/install.yml b/ansible/roles/warden-server/tasks/install.yml index 7ba8c3b..58fb38d 100644 --- a/ansible/roles/warden-server/tasks/install.yml +++ b/ansible/roles/warden-server/tasks/install.yml @@ -1,16 +1,10 @@ - name: Install Warden rpm dependencies yum: "name={{ packages }} state=installed" -- name: Update pip2 - pip: - name: "{{ pip_update }}" - executable: pip2 - state: latest - - name: Install Warden pip dependencies pip: name: "{{ pip_packages }}" - executable: pip2 + executable: pip3 - name: Start & enable MariaDB service: name=mariadb state=started enabled=yes diff --git a/ansible/roles/warden-server/vars/main.yml b/ansible/roles/warden-server/vars/main.yml index aef55f1..b3b30f0 100644 --- a/ansible/roles/warden-server/vars/main.yml +++ b/ansible/roles/warden-server/vars/main.yml @@ -1,6 +1,5 @@ -packages: [mariadb-server, mariadb-devel, mod_ssl, mod_wsgi, swig, openssl-devel, gcc, python-pip, python-devel] -pip_update: [pip, setuptools] -pip_packages: ["ZEO==4.1.0", zope.security==4.1.0, mysqldbda, mysql, M2Crypto, jsonschema] +packages: [mariadb-server, mariadb-devel, httpd-devel, mod_ssl, swig, openssl-devel, gcc,] +pip_packages: ["ZEO==4.1.0", zope.security==4.1.0, mod-wsgi, mysqldbda, mysql, M2Crypto, jsonschema] warden_install_dir: { src: "install/", diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 1a52220..ca7b833 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -4,7 +4,7 @@ Vagrant.configure(2) do |config| list.vm.box = "centos/7" list.vm.provider :virtualbox do |v| - v.customize ["modifyvm", :id, "--memory", 1024] + v.customize ["modifyvm", :id, "--memory", 2048] v.customize ["modifyvm", :id, "--cpus", 1] v.customize ["modifyvm", :id, "--name", "list-vagrant"] end @@ -25,6 +25,11 @@ Vagrant.configure(2) do |config| ansible.playbook = "../ansible/site.yml" ansible.inventory_path = "../ansible/inventory/hosts" ansible.tags = "install" + ansible.raw_arguments = [ + "--skip-tags=nagios-client,nagios-server,letsencrypt", + "-vv", + ] + end end