diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml index 55f38f8f1..d765af2bc 100644 --- a/roles/haproxy/tasks/main.yml +++ b/roles/haproxy/tasks/main.yml @@ -16,7 +16,7 @@ - name: Install haproxy and socat ansible.builtin.apt: name: - - "haproxy=3.0.*" + - "haproxy" - "socat" - "git" state: "present" @@ -88,17 +88,6 @@ group: haproxy mode: "0770" -- name: Create combined key and certificate file for HAproxy - ansible.builtin.copy: - content: > - {{ item.key_content }}{{ lookup('file', '{{ inventory_dir }}/files/certs/{{ item.crt_name }}') }} - dest: "/etc/haproxy/certs/{{ item.name }}_haproxy.pem" - mode: "0600" - with_items: "{{ haproxy_sni_ip.certs }}" - when: haproxy_sni_ip.certs is defined - notify: - - "reload haproxy" - - name: Create backend CA directory ansible.builtin.file: path: "{{ tls_backend_ca | dirname }}" diff --git a/roles/haproxy/templates/certlist.lst.j2 b/roles/haproxy/templates/certlist.lst.j2 index 3e8bb226d..800a79b39 100644 --- a/roles/haproxy/templates/certlist.lst.j2 +++ b/roles/haproxy/templates/certlist.lst.j2 @@ -3,11 +3,6 @@ /etc/haproxy/certs/{{ host }}.pem [ocsp-update on] {% endfor %} {% endif %} -{% if haproxy_sni_ip.certs is defined %} -{% for cert in haproxy_sni_ip.certs %} -/etc/haproxy/certs/{{ cert.name }}_haproxy.pem [ocsp-update on] -{% endfor %} -{% endif %} {% if haproxy_extra_certs is defined %} {% for cert in haproxy_extra_certs %} {{ cert }} [ocsp-update on] diff --git a/roles/haproxy/templates/update_ocsp.j2 b/roles/haproxy/templates/update_ocsp.j2 deleted file mode 100644 index 2ed61f528..000000000 --- a/roles/haproxy/templates/update_ocsp.j2 +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# Call hapos-upd to update OCSP stapling info foreach of our haproxy certificates - -# probably we want to continue even if one fails -set -e - -{% for cert in haproxy_sni_ip.certs %} -/usr/local/sbin/hapos-upd --partial-chain --good-only --socket /var/lib/haproxy/haproxy.stats \ - --VAfile /etc/pki/haproxy/{{ cert.name }}_haproxy.pem \ - --cert /etc/pki/haproxy/{{ cert.name }}_haproxy.pem -{% endfor %} diff --git a/roles/manage/tasks/main.yml b/roles/manage/tasks/main.yml index 9df3ecb97..ea54859f3 100644 --- a/roles/manage/tasks/main.yml +++ b/roles/manage/tasks/main.yml @@ -11,6 +11,15 @@ - "/opt/openconext/manage/metadata_templates" - "/opt/openconext/manage/policies" +- name: Copy Stepup stepup_config.json from inventory + ansible.builtin.copy: + src: "{{ inventory_dir }}/files/manage/stepup_config.json" + dest: "/opt/openconext/manage/stepup_config.json" + owner: "root" + group: "root" + mode: "0644" + notify: restart manageserver + - name: Import the mongo CA file ansible.builtin.copy: src: "{{ inventory_dir }}/secrets/mongo/mongoca.pem" @@ -114,6 +123,10 @@ - source: /opt/openconext/manage/__cacert_entrypoint.sh target: /__cacert_entrypoint.sh type: bind + - source: /opt/openconext/manage/stepup_config.json + target: /stepup_config.json + type: bind + command: "java -jar /app.jar -Xmx512m --spring.config.location=./config/" etc_hosts: host.docker.internal: host-gateway diff --git a/roles/manage/templates/application.yml.j2 b/roles/manage/templates/application.yml.j2 index 5b7a7980d..31085a6d7 100644 --- a/roles/manage/templates/application.yml.j2 +++ b/roles/manage/templates/application.yml.j2 @@ -53,11 +53,20 @@ push: user: {{ pdp.username }} password: "{{ pdp.password }}" enabled: {{ manage.pdp_push_enabled }} + stepup: + url: https://middleware.{{ base_domain }} + user: {{ manage.middleware_user }} + configuration_file: "file:///stepup_config.json" + password: {{ manage_middleware_password }} + enabled: {{ manage.stepup_push_enabled }} + product: name: Manage organization: {{ instance_name }} service_provider_feed_url: {{ manage_service_provider_feed_url }} + jira_base_url: https://servicedesk.surf.nl/jira/browse/ + jira_ticket_prefixes: CXT,SD supported_languages: {{ supported_language_codes }} show_oidc_rp: {{ manage_show_oidc_rp_tab }} @@ -84,7 +93,7 @@ policies: extra_saml_attributes: file://{{ manage_dir }}/policies/extra_saml_attributes.json sram: - sram_rp_entity_id: ""{{ manage.sram_rp_entity_id }}" + sram_rp_entity_id: "{{ manage.sram_rp_entity_id }}" spring: mail: diff --git a/roles/myconext/templates/application.yml.j2 b/roles/myconext/templates/application.yml.j2 index 6af4e6ae9..9e2af3722 100644 --- a/roles/myconext/templates/application.yml.j2 +++ b/roles/myconext/templates/application.yml.j2 @@ -106,9 +106,11 @@ mobile_app_rp_entity_id: {{ myconext.mobile_app_rp_entity_id }} create-from-institution: return-url-allowed-domains: - {% for url in create_from_institution_return_url_allowed_domains %} +{% for url in myconext.create_from_institution_return_url_allowed_domains | default([]) %} - "{{ url }}" - {% endfor %} +{% else %} + [] # lege lijst wanneer er geen URLs zijn +{% endfor %} # The host headers to identify the service the user is logged in host_headers: diff --git a/roles/rsyslog/tasks/main.yml b/roles/rsyslog/tasks/main.yml index 1fc0608dc..a531fd677 100644 --- a/roles/rsyslog/tasks/main.yml +++ b/roles/rsyslog/tasks/main.yml @@ -1,9 +1,10 @@ -- name: Install rsyslog +- name: Install rsyslog and python modules ansible.builtin.package: name: - rsyslog - rsyslog-gnutls - rsyslog-relp + - python3-dateutil state: present notify: - "restart rsyslog" diff --git a/roles/rsyslog/tasks/process_auth_logs.yml b/roles/rsyslog/tasks/process_auth_logs.yml index e62027530..804bf629b 100644 --- a/roles/rsyslog/tasks/process_auth_logs.yml +++ b/roles/rsyslog/tasks/process_auth_logs.yml @@ -39,7 +39,7 @@ state: present when: ansible_os_family == "Debian" -- name: Create a python script that parses log_logins per environment +- name: Create a python script that parses eb log_logins per environment ansible.builtin.template: src: parse_ebauth_to_mysql.py.j2 dest: /usr/local/sbin/parse_ebauth_to_mysql_{{ item.name }}.py @@ -49,7 +49,17 @@ with_items: "{{ rsyslog_environments }}" when: item.db_loglogins_name is defined -- name: Put log_logins logrotate scripts +- name: Create a python script that parses stepup log_logins per environment + ansible.builtin.template: + src: parse_stepupauth_to_mysql.py.j2 + dest: /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py + mode: 0740 + owner: root + group: root + with_items: "{{ rsyslog_environments }}" + when: item.db_loglogins_name is defined + +- name: Put log_logins logrotate scripts for eb ansible.builtin.template: src: logrotate_ebauth.j2 dest: /etc/logrotate.d/logrotate_ebauth_{{ item.name }} @@ -59,6 +69,16 @@ with_items: "{{ rsyslog_environments }}" when: item.db_loglogins_name is defined +- name: Put log_logins logrotate scripts for stepup + ansible.builtin.template: + src: logrotate_stepupauth.j2 + dest: /etc/logrotate.d/logrotate_stepupauth_{{ item.name }} + mode: 0644 + owner: root + group: root + with_items: "{{ rsyslog_environments }}" + when: item.db_loglogins_name is defined + - name: Create logdirectory for log_logins cleanup script ansible.builtin.file: path: "{{ rsyslog_dir }}/apps/{{ item.name }}/loglogins_cleanup/" diff --git a/roles/rsyslog/templates/logrotate_stepupauth.j2 b/roles/rsyslog/templates/logrotate_stepupauth.j2 new file mode 100644 index 000000000..be1a50652 --- /dev/null +++ b/roles/rsyslog/templates/logrotate_stepupauth.j2 @@ -0,0 +1,16 @@ +{{ rsyslog_dir }}/log_logins/{{ item.name }}/stepup-authentication.log +{ + missingok + daily + rotate 180 + sharedscripts + dateext + dateyesterday + compress + delaycompress + create 0640 root {{ rsyslog_read_group }} + postrotate + /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py > /dev/null + systemctl kill -s HUP rsyslog.service + endscript +} diff --git a/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 index b37f4720c..7e0bc7bcb 100644 --- a/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 @@ -21,11 +21,17 @@ cursor = db.cursor() def update_lastseen(user_id, date): query = """ - REPLACE INTO last_login (userid, lastseen) + INSERT INTO last_login (userid, lastseen) VALUES (%s, %s) + ON DUPLICATE KEY UPDATE + lastseen = GREATEST(lastseen, VALUES(lastseen)) """ - cursor.execute(query, (user_id, date)) - db.commit() + try: + cursor.execute(query, (user_id, date)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error updating last_login for user {user_id}: {e}") def load_in_mysql(a,b,c,d,e,f,g,h): sql = """insert into log_logins(idpentityid,spentityid,loginstamp,userid,keyid,sessionid,requestid,trustedproxyentityid) values(%s,%s,%s,%s,%s,%s,%s,%s)""" @@ -73,4 +79,3 @@ for filename in os.listdir(workdir): cursor.close() db.close() - diff --git a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 new file mode 100644 index 000000000..843fe44bc --- /dev/null +++ b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 @@ -0,0 +1,152 @@ +#!/usr/bin/python3 +# This script parses rotated stepup-authentication.log files produced by engineblock. +# It filters for successful logins (authentication_result:OK) and inserts the data +# into the log_logins and last_login MySQL tables. +# This script is intended to be run separately during logrotate. + +import os +import sys +import json +import MySQLdb +from dateutil.parser import parse + +# Configuration variables (to be injected by Ansible/Jinja2) +mysql_host="{{ item.db_loglogins_host }}" +mysql_user="{{ item.db_loglogins_user }}" +mysql_password="{{ item.db_loglogins_password }}" +mysql_db="{{ item.db_loglogins_name }}" +workdir="{{ rsyslog_dir }}/log_logins/{{ item.name}}/" + +# Establish database connection +try: + db = MySQLdb.connect(mysql_host,mysql_user,mysql_password,mysql_db ) + cursor = db.cursor() +except Exception as e: + print(f"Error connecting to MySQL: {e}") + sys.exit(1) + +# --- Database Functions --- + +def update_lastseen(user_id, date): + """ + Updates the last_login table. + Uses GREATEST() to ensure only newer dates overwrite the existing 'lastseen' value. + """ + query = """ + INSERT INTO last_login (userid, lastseen) + VALUES (%s, %s) + ON DUPLICATE KEY UPDATE + lastseen = GREATEST(lastseen, VALUES(lastseen)) + """ + try: + cursor.execute(query, (user_id, date)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error updating last_login for user {user_id}: {e}") + +def load_stepup_in_mysql(idp, sp, loginstamp, userid, requestid): + """ + Inserts Step-up login data into the log_logins table. + Fills keyid, sessionid, and trustedproxyentityid with NULL. + """ + # Columns in log_logins: idpentityid, spentityid, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid + + keyid = None + sessionid = None + trustedproxyentityid = None + + sql = """ + INSERT INTO log_logins(idpentityid, spentityid, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid) + VALUES(%s, %s, %s, %s, %s, %s, %s, %s) + """ + try: + cursor.execute(sql, (idp, sp, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error inserting stepup data: {e}") + # Print the data that failed insertion + print((idp, sp, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid)) + +# --- Parsing Function --- + +def parse_stepup_lines(a): + """ + Opens the stepup log file, parses each line, filters for successful logins, + and loads the data into MySQL. + """ + input_file = open((a), 'r') + for line in input_file: + try: + # Assumes JSON data starts after the first ']:' + jsonline = line.split(']:',2)[1] + data = json.loads(jsonline) + except: + continue + + # 1. Filtering condition: Only parse logs having authentication_result:OK + # Only successful authentications are logged, so this check is not + # necessary. There is currently a bug in the Stepup-Gateway where + # FAILED is logged, even though the result is OK, making this check + # do the wrong thing now. + # + #if data.get("context").("authentication_result") != "OK": + # continue + + # 2. Extract required fields + context = data.get("context") + + if not isinstance(context, dict): + print("Skipping line: context is missing or invalid") + continue + + user_id = context.get("identity_id") + timestamp = context.get("datetime") + request_id = context.get("request_id") + sp_entity_id = context.get("requesting_sp") + idp_entity_id = context.get("authenticating_idp") + + # Basic data validation + if not user_id or not timestamp: + print( + "Skipping line: validation failed " + f"(user_id={user_id!r}, timestamp={timestamp!r}, request_id={request_id!r})" + ) + continue + + try: + # 3. Format date and time for MySQL + loginstamp = parse(timestamp).strftime("%Y-%m-%d %H:%M:%S") + last_login_date = parse(timestamp).strftime("%Y-%m-%d") + except: + print( + "Skipping line: timestamp parsing failed " + f"(timestamp={timestamp!r}, user_id={user_id!r}, error={e})" + ) + continue + + # 4. Insert into MySQL + load_stepup_in_mysql(idp_entity_id, sp_entity_id, loginstamp, user_id, request_id) + + # 5. Update last login date + update_lastseen(user_id, last_login_date) + + +# --- Main Execution --- + +## Loop over the files and parse them one by one +for filename in os.listdir(workdir): + filetoparse=(os.path.join(workdir, filename)) + + # Check for Stepup files, ignore compressed files + if os.path.isfile(filetoparse) and filename.startswith("stepup-authentication.log-") and not filename.endswith(".gz"): + print(f"Parsing stepup log file: {filename}") + parse_stepup_lines(filetoparse) + else: + continue + +# Close database connection +cursor.close() +db.close() +print("Stepup log parsing complete.")