From 8ad0f8aaa7af1ee35b712d095baded5842a4d4da Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 21 Nov 2025 00:05:43 +0000 Subject: [PATCH 1/3] feat: add support for enabling `Pulp` TLS Add playbooks, configuration and documentation to support the deployment of Pulp TLS. --- doc/source/configuration/openbao.rst | 34 +++++++++++++ .../ansible/deployment/copy-ca-to-hosts.yml | 47 ++++++++++++++++++ .../pulp/pulp-generate-certificate.yml | 49 +++++++++++++++++++ etc/kayobe/container-engine.yml | 2 +- etc/kayobe/pulp.yml | 4 +- .../notes/pulp-tls-105e47f0da602a25.yaml | 5 ++ 6 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 etc/kayobe/ansible/deployment/copy-ca-to-hosts.yml create mode 100644 etc/kayobe/ansible/pulp/pulp-generate-certificate.yml create mode 100644 releasenotes/notes/pulp-tls-105e47f0da602a25.yaml diff --git a/doc/source/configuration/openbao.rst b/doc/source/configuration/openbao.rst index 78c5011042..b7ef4949cb 100644 --- a/doc/source/configuration/openbao.rst +++ b/doc/source/configuration/openbao.rst @@ -460,6 +460,40 @@ Enable the required TLS variables in kayobe and kolla kayobe overcloud host command run --command "systemctl restart kolla-nova_compute-container.service" --become --show-output -l compute +Pulp TLS +======== + +.. warning:: + + For clouds in production consider the impact of enabling TLS on specific hosts as Docker daemon will be restarted and this will disrupt deployments of Ceph Reef and older. + +To enable TLS for Pulp we first need to generate the certificates and the proceed to configure all hosts that use Pulp to add the root CA to their truststore. + +1. Generate the certificate + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp/pulp-generate-certificate.yml + +2. Copy CA to truststore + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deployment/copy-ca-to-hosts.yml + +3. Enable TLS for Pulp in pulp.yml + + .. code-block:: + + # Whether to enable TLS for Pulp. + pulp_enable_tls: true + +4. Redeploy Pulp + + .. code-block:: + + kayobe seed service reconfigure -t seed-deploy-containers -kt none + Barbican integration ==================== diff --git a/etc/kayobe/ansible/deployment/copy-ca-to-hosts.yml b/etc/kayobe/ansible/deployment/copy-ca-to-hosts.yml new file mode 100644 index 0000000000..e0a03c95e9 --- /dev/null +++ b/etc/kayobe/ansible/deployment/copy-ca-to-hosts.yml @@ -0,0 +1,47 @@ +--- +- name: Install certificate authorities and update trust + hosts: overcloud:seed:seed-hypervisor + # Avoid using facts because this may be used as a pre overcloud host + # configure hook, and we don't want to populate the fact cache (if one is in + # use) with the bootstrap user's context. + gather_facts: false + tags: + - install-ca + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: -o StrictHostKeyChecking=no + # Don't assume facts are present. + os_family: "{{ ansible_facts.os_family | default('Debian' if os_distribution == 'ubuntu' else 'RedHat') }}" + become: true + tasks: + - name: Install certificate authorities on RedHat based distributions + when: os_family == 'RedHat' + block: + - name: Copy certificate authorities on RedHat family systems (Rocky, RHEL, CentOS) + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/openbao/{{ item }}.pem" + dest: "/etc/pki/ca-trust/source/anchors/{{ item }}.crt" + mode: "0644" + loop: + - "OS-TLS-ROOT" + + - name: Update CA trust on RedHat family systems + ansible.builtin.command: "update-ca-trust" + + - name: Install certificate authorities on Debian based distributions + when: os_family == 'Debian' + block: + - name: Copy certificate authorities on Debian family systems (Ubuntu, Debian) + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/openbao/{{ item }}.pem" + dest: "/usr/local/share/ca-certificates/{{ item }}.crt" + mode: "0644" + loop: + - "OS-TLS-ROOT" + + - name: Update CA trust on Debian family systems + ansible.builtin.command: "update-ca-certificates" diff --git a/etc/kayobe/ansible/pulp/pulp-generate-certificate.yml b/etc/kayobe/ansible/pulp/pulp-generate-certificate.yml new file mode 100644 index 0000000000..ba7fecb9f3 --- /dev/null +++ b/etc/kayobe/ansible/pulp/pulp-generate-certificate.yml @@ -0,0 +1,49 @@ +--- +- name: Generate certificates + hosts: seed + run_once: true + vars: + openbao_api_addr: http://127.0.0.1:8200 + openbao_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/seed-openbao-keys.json" + name: openbao_keys + + - name: Issue a certificate Pulp + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ openbao_keys.root_token }}" + mount_point: "{{ openbao_intermediate_ca_name }}" + role: "{{ overcloud_openbao_pki_default_role_name }}" + common_name: "{{ inventory_hostname }}" + extra_params: + ip_sans: "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" + register: pulp_certificate + + - name: Ensure pulp certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/pulp/certificates" + state: directory + delegate_to: localhost + + - name: Write certificate to file + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/pulp/certificates/pulp.crt" + content: | + {{ pulp_certificate.data.certificate }} + {{ pulp_certificate.data.issuing_ca }} + mode: "0600" + delegate_to: localhost + + - name: Write key to file + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/pulp/certificates/pulp.key" + content: | + {{ pulp_certificate.data.private_key }} + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/container-engine.yml b/etc/kayobe/container-engine.yml index 02f50de623..292de969b5 100644 --- a/etc/kayobe/container-engine.yml +++ b/etc/kayobe/container-engine.yml @@ -40,7 +40,7 @@ docker_registry: "{{ stackhpc_docker_registry }}" docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" # CA of docker registry -#docker_registry_ca: +docker_registry_ca: "{{ kayobe_env_config_path ~ '/openbao/OS-TLS-INT.crt' if pulp_enable_tls | bool else none }}" # List of Docker registry mirrors. #docker_registry_mirrors: diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 845b452c4c..409e58c0d6 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -14,10 +14,10 @@ pulp_port: "{{ '443' if pulp_enable_tls | bool else '80' }}" pulp_enable_tls: false # Path to a TLS certificate to use when TLS is enabled. -#pulp_cert_path: +pulp_cert_path: "{{ kayobe_env_config_path ~ '/pulp/certificates/pulp.crt' if pulp_enable_tls | bool else '' }}" # Path to a TLS key to use when TLS is enabled. -#pulp_key_path: +pulp_key_path: "{{ kayobe_env_config_path ~ '/pulp/certificates/pulp.key' if pulp_enable_tls | bool else '' }}" ############################################################################### # Local Pulp access credentials diff --git a/releasenotes/notes/pulp-tls-105e47f0da602a25.yaml b/releasenotes/notes/pulp-tls-105e47f0da602a25.yaml new file mode 100644 index 0000000000..a92992c008 --- /dev/null +++ b/releasenotes/notes/pulp-tls-105e47f0da602a25.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add playbooks and configuration to enable the easy deployment of Pulp with + TLS support. From ad1fead5b6c7a2189c22c404cce12217bfa523e0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 25 Nov 2025 12:56:43 +0000 Subject: [PATCH 2/3] docs: add suggested changes - Clean up name of tasks - Add OpenBao assertion - Update release note Signed-off-by: Jack Hodgkiss --- doc/source/configuration/openbao.rst | 4 +++- etc/kayobe/ansible/pulp/pulp-generate-certificate.yml | 10 ++++++++-- releasenotes/notes/pulp-tls-105e47f0da602a25.yaml | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/doc/source/configuration/openbao.rst b/doc/source/configuration/openbao.rst index b7ef4949cb..73b8cecb30 100644 --- a/doc/source/configuration/openbao.rst +++ b/doc/source/configuration/openbao.rst @@ -466,6 +466,7 @@ Pulp TLS .. warning:: For clouds in production consider the impact of enabling TLS on specific hosts as Docker daemon will be restarted and this will disrupt deployments of Ceph Reef and older. + As Vault is deprecated and will be removed in future releases this process only works for OpenBao To enable TLS for Pulp we first need to generate the certificates and the proceed to configure all hosts that use Pulp to add the root CA to their truststore. @@ -481,9 +482,10 @@ To enable TLS for Pulp we first need to generate the certificates and the procee kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deployment/copy-ca-to-hosts.yml -3. Enable TLS for Pulp in pulp.yml +3. Enable TLS for Pulp .. code-block:: + :caption: $KAYOBE_CONFIG_PATH/pulp.yml # Whether to enable TLS for Pulp. pulp_enable_tls: true diff --git a/etc/kayobe/ansible/pulp/pulp-generate-certificate.yml b/etc/kayobe/ansible/pulp/pulp-generate-certificate.yml index ba7fecb9f3..3c2f258894 100644 --- a/etc/kayobe/ansible/pulp/pulp-generate-certificate.yml +++ b/etc/kayobe/ansible/pulp/pulp-generate-certificate.yml @@ -6,12 +6,18 @@ openbao_api_addr: http://127.0.0.1:8200 openbao_intermediate_ca_name: OS-TLS-INT tasks: + - name: Assert that stackhpc_ca_secret_store is 'openbao' + ansible.builtin.assert: + that: + - stackhpc_ca_secret_store == "openbao" + fail_msg: "stackhpc_ca_secret_store must be 'openbao'" + - name: Include OpenBao keys ansible.builtin.include_vars: file: "{{ kayobe_env_config_path }}/openbao/seed-openbao-keys.json" name: openbao_keys - - name: Issue a certificate Pulp + - name: Issue Pulp certificate hashivault_pki_cert_issue: # noqa: fqcn url: "{{ openbao_api_addr }}" ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" @@ -23,7 +29,7 @@ ip_sans: "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" register: pulp_certificate - - name: Ensure pulp certificates directory exists + - name: Ensure Pulp certificates directory exists ansible.builtin.file: path: "{{ kayobe_env_config_path }}/pulp/certificates" state: directory diff --git a/releasenotes/notes/pulp-tls-105e47f0da602a25.yaml b/releasenotes/notes/pulp-tls-105e47f0da602a25.yaml index a92992c008..298bc00353 100644 --- a/releasenotes/notes/pulp-tls-105e47f0da602a25.yaml +++ b/releasenotes/notes/pulp-tls-105e47f0da602a25.yaml @@ -2,4 +2,4 @@ features: - | Add playbooks and configuration to enable the easy deployment of Pulp with - TLS support. + TLS support in combination with certificates generated via OpenBao. From ba1cb202c708aaa5febc6b805874701893433f55 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 29 Nov 2025 14:16:27 +0000 Subject: [PATCH 3/3] fix: remove `docker_registry_ca` from being set The task that uses this variable cannot handle the variable being empty or omitted. This has been left unset within the base environment. Signed-off-by: Jack Hodgkiss --- doc/source/configuration/openbao.rst | 14 ++++++++++++++ etc/kayobe/container-engine.yml | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/source/configuration/openbao.rst b/doc/source/configuration/openbao.rst index 73b8cecb30..f82f8b6b37 100644 --- a/doc/source/configuration/openbao.rst +++ b/doc/source/configuration/openbao.rst @@ -496,6 +496,20 @@ To enable TLS for Pulp we first need to generate the certificates and the procee kayobe seed service reconfigure -t seed-deploy-containers -kt none +5. Set CA for docker registry + + .. code-block:: + :caption: $KAYOBE_CONFIG_PATH/container-engine.yml + + # CA of docker registry + docker_registry_ca: "{{ kayobe_env_config_path ~ '/openbao/OS-TLS-INT.crt' if pulp_enable_tls | bool else '' }}" + +6. Perform host configure to reconfigure APT, DNF and docker/podman settings + + .. code-block:: + + kayobe overcloud host configure -t dnf,apt,container-engine + Barbican integration ==================== diff --git a/etc/kayobe/container-engine.yml b/etc/kayobe/container-engine.yml index 292de969b5..02f50de623 100644 --- a/etc/kayobe/container-engine.yml +++ b/etc/kayobe/container-engine.yml @@ -40,7 +40,7 @@ docker_registry: "{{ stackhpc_docker_registry }}" docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" # CA of docker registry -docker_registry_ca: "{{ kayobe_env_config_path ~ '/openbao/OS-TLS-INT.crt' if pulp_enable_tls | bool else none }}" +#docker_registry_ca: # List of Docker registry mirrors. #docker_registry_mirrors: