Update CI files from main branch

.ci/ansible/Containerfile.j2

@@ -20,14 +20,21 @@ RUN pip3 install \
 {%- if azure_test | default(false) -%}
 {{ " " }}django-storages[azure]>=1.12.2
 {%- endif -%}
+{%- if gcp_test | default(false) -%}
+{{ " " }}django-storages[google]>=1.13.2
+{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm
 {%- endif -%}
 {{ " " }}"{{ item.source }}"
 {%- endfor %}
 
-RUN mkdir -p /etc/nginx/pulp/
+USER pulp:pulp
+RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
+       /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link
+USER root:root
+
 {% for item in plugins %}
 RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
     ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true

.ci/ansible/settings.py.j2

@@ -8,7 +8,7 @@ TOKEN_SIGNATURE_ALGORITHM = "ES256"
 CACHE_ENABLED = True
 REDIS_HOST = "localhost"
 REDIS_PORT = 6379
-TELEMETRY = False
+ANALYTICS = False
 
 {% if api_root is defined %}
 API_ROOT = {{ api_root | repr }}
@@ -63,3 +63,10 @@ AZURE_OVERWRITE_FILES = True
 AZURE_URL_EXPIRATION_SECS = 120
 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;'
 {% endif %}
+
+{% if gcp_test | default(false) %}
+DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
+MEDIA_ROOT = ""
+GS_BUCKET_NAME = "gcppulp"
+GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
+{% endif %}

.ci/ansible/start_container.yaml

@@ -19,6 +19,8 @@
       community.crypto.openssh_keypair:
         path: ssh/id_ed25519
         type: ed25519
+        owner: 700  # pulp in the container
+      become: true
       when: stream_test | default(false)
 
     - name: "Generate Pulp Settings"

.ci/assets/release_requirements.txt

@@ -1,4 +1,3 @@
 bump2version
 gitpython
-python-redmine
 towncrier

.ci/scripts/calc_deps_lowerbounds.py

@@ -0,0 +1,41 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_file' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+from packaging.requirements import Requirement
+
+
+# We install these from source, but it would be really handy if we could test pulpcore
+# compatibility that way too.
+EXCEPTIONS = [
+    "pulpcore",
+    "pulp-certguard".replace("-", "_"),
+]
+
+
+def main():
+    """Calculate the lower bound of dependencies where possible."""
+    with open("requirements.txt") as req_file:
+        for line in req_file:
+            try:
+                requirement = Requirement(line)
+            except ValueError:
+                print(line.strip())
+            else:
+                if requirement.name.replace("-", "_") in EXCEPTIONS:
+                    print(line.strip())
+                else:
+                    for spec in requirement.specifier:
+                        if spec.operator == ">=":
+                            min_version = str(spec)[2:]
+                            print(f"{requirement.name}=={min_version}")
+                            break
+                    else:
+                        print(line.strip())
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/check_gettext.sh

@@ -12,7 +12,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../..
 
 set -uv
 
-# check for imports not from pulpcore.plugin. exclude tests
 MATCHES=$(grep -n -r --include \*.py "_(f")
 
 if [ $? -ne 1 ]; then

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-174-g56e0727
+2021.08.26-197-g21230dd

.github/workflows/ci.yml

@@ -98,11 +98,13 @@ jobs:
           - TEST: azure
           - TEST: s3
           - TEST: stream
+          - TEST: lowerbounds
     outputs:
       deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }}
       deprecations-stream: ${{ steps.deprecations.outputs.deprecations-stream }}
       deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }}
       deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }}
+      deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }}
 
     steps:
       - uses: actions/checkout@v3
@@ -196,7 +198,7 @@ jobs:
 
       - name: Extract Deprecations from Logs
         id: deprecations
-        run: echo "::set-output name=deprecations-${{ matrix.env.TEST }}::$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0)"
+        run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT
 
       - name: Logs
         if: always()
@@ -220,12 +222,14 @@ jobs:
           test -z "${{ needs.test.outputs.deprecations-stream }}"
           test -z "${{ needs.test.outputs.deprecations-azure }}"
           test -z "${{ needs.test.outputs.deprecations-s3 }}"
+          test -z "${{ needs.test.outputs.deprecations-lowerbounds }}"
       - name: Print deprecations
         if: failure()
         run: |
           echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-stream }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d
+          echo "${{ needs.test.outputs.deprecations-lowerbounds }}" | base64 -d
 
 

.github/workflows/nightly.yml

@@ -27,8 +27,10 @@ jobs:
           - TEST: azure
           - TEST: s3
           - TEST: stream
+
           - TEST: generate-bindings
           - TEST: plugin-from-pypi
+          - TEST: lowerbounds
 
     steps:
       - uses: actions/checkout@v3
@@ -312,32 +314,6 @@ jobs:
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
 
 
-      - name: Download Ruby client
-        uses: actions/download-artifact@v3
-        with:
-          name: ruby-client.tar
-
-      - name: Untar Ruby client packages
-        run: tar -xvf ruby-client.tar
-
-      - name: Publish client to rubygems
-        run: bash .github/workflows/scripts/publish_client_gem.sh
-
-
-
-      - name: Download Python client
-        uses: actions/download-artifact@v3
-        with:
-          name: python-client.tar
-
-      - name: Untar python client packages
-        run: tar -xvf python-client.tar
-
-      - name: Publish client to pypi
-        run: bash .github/workflows/scripts/publish_client_pypi.sh
-
-
-
       - name: Download built docs
         uses: actions/download-artifact@v3
         with:

.github/workflows/release.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Install python dependencies
         run: |
           echo ::group::PYDEPS
-          pip install bandersnatch bump2version gitpython python-redmine towncrier==19.9.0 wheel
+          pip install packaging~=21.3 bandersnatch bump2version gitpython towncrier==19.9.0 wheel
           echo ::endgroup::
 
       - name: Configure Git with pulpbot name and email
@@ -86,6 +86,7 @@ jobs:
           - TEST: s3
           - TEST: stream
           - TEST: generate-bindings
+          - TEST: lowerbounds
 
     steps:
       - uses: actions/download-artifact@v3
@@ -293,7 +294,7 @@ jobs:
       - name: Install python dependencies
         run: |
           echo ::group::PYDEPS
-          pip install gitpython python-redmine requests packaging tweepy
+          pip install gitpython requests packaging~=21.3 tweepy
           echo ::endgroup::
 
       - name: Push branch and tag to GitHub
@@ -335,8 +336,6 @@ jobs:
       - name: Publish client to rubygems
         run: bash .github/workflows/scripts/publish_client_gem.sh
 
-
-
       - name: Update GitHub
         continue-on-error: true
         run: |

.github/workflows/scripts/before_install.sh

@@ -95,17 +95,19 @@ if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then
 fi
 
 
-git clone --depth=1 https://github.com/pulp/pulp-cli.git
+git clone https://github.com/pulp/pulp-cli.git
+cd pulp-cli
 if [ -n "$PULP_CLI_PR_NUMBER" ]; then
-  cd pulp-cli
   git fetch origin pull/$PULP_CLI_PR_NUMBER/head:$PULP_CLI_PR_NUMBER
   git checkout $PULP_CLI_PR_NUMBER
-  cd ..
+  pip install . ./pulp-glue
+else
+  pip install pulp-cli
+  PULP_CLI_VERSION="$(python -c "from pulpcore.cli.common import __version__; print(__version__)")"
+  git checkout "$PULP_CLI_VERSION"
 fi
 
-cd pulp-cli
-pip install .
-pulp config create --base-url https://pulp --location tests/cli.toml 
+pulp config create --base-url https://pulp  --location tests/cli.toml
 mkdir ~/.config/pulp
 cp tests/cli.toml ~/.config/pulp/cli.toml
 cd ..
@@ -147,6 +149,11 @@ fi
 
 cd pulp_file
 
+if [[ "$TEST" = "lowerbounds" ]]; then
+  python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt
+  mv lowerbounds_requirements.txt requirements.txt
+fi
+
 if [ -f $POST_BEFORE_INSTALL ]; then
   source $POST_BEFORE_INSTALL
 fi

.github/workflows/scripts/before_script.sh

@@ -29,10 +29,14 @@ tail -v -n +1 .ci/ansible/vars/main.yaml
 echo "PULP CONFIG:"
 tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 
-SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings")
+# Needed for some functional tests
+cmd_prefix bash -c "echo '%wheel        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
+cmd_prefix bash -c "usermod -a -G wheel pulp"
+
+SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds")
 if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
   # Many functional tests require these
-  cmd_prefix dnf install -yq lsof which dnf-plugins-core
+  cmd_prefix dnf install -yq lsof which
 fi
 
 if [[ "${REDIS_DISABLED:-false}" == true ]]; then

.github/workflows/scripts/install.sh

@@ -79,7 +79,7 @@ services:
     volumes:
       - ./settings:/etc/pulp
       - ./ssh:/keys/
-      - ~/.config:/root/.config
+      - ~/.config:/var/lib/pulp/.config
       - ../../../pulp-openapi-generator:/root/pulp-openapi-generator
     env:
       PULP_WORKERS: "4"
@@ -93,12 +93,14 @@ pulp_container_tag: https
 
 VARSYAML
 
-SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings")
+SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds")
 if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
   sed -i -e '/^services:/a \
   - name: pulp-fixtures\
     image: docker.io/pulp/pulp-fixtures:latest\
     env: {BASE_URL: "http://pulp-fixtures:8080"}' vars/main.yaml
+
+  export REMOTE_FIXTURES_ORIGIN="http://pulp-fixtures:8080"
 fi
 if [ "$TEST" == 'stream' ]; then
   sed -i -e '/^services:/a \
@@ -156,6 +158,15 @@ fi
 
 ansible-playbook build_container.yaml
 ansible-playbook start_container.yaml
+
+# .config needs to be accessible by the pulp user in the container, but some
+# files will likely be modified on the host by post/pre scripts.
+chmod 777 ~/.config/pulp_smash/
+chmod 666 ~/.config/pulp_smash/settings.json
+sudo chown -R 700:700 ~runner/.config
+# Plugins often write to ~/.config/pulp/cli.toml from the host
+sudo chmod 777 ~runner/.config/pulp
+sudo chmod 666 ~runner/.config/pulp/cli.toml
 echo ::group::SSL
 # Copy pulp CA
 sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt

.github/workflows/scripts/release.py

@@ -79,9 +79,7 @@ def create_release_commits(repo, release_version, plugin_path):
     git.add(f"{plugin_path}/setup.py")
     git.add(f"{plugin_path}/requirements.txt")
     git.add(f"{plugin_path}/.bumpversion.cfg")
-
     git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]")
-
     sha = repo.head.object.hexsha
     short_sha = git.rev_parse(sha, short=7)
 
@@ -101,7 +99,6 @@ def create_release_commits(repo, release_version, plugin_path):
     git.add(f"{plugin_path}/requirements.txt")
     git.add(f"{plugin_path}/.bumpversion.cfg")
     git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]")
-
     print(f"Release commit == {short_sha}")
     print(f"All changes were committed on branch: release_{release_version}")
     return sha
@@ -122,8 +119,8 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path):
                 tag = existing_tag
             else:
                 raise RuntimeError(
-                    "The '{desired_tag}' tag already exists, but the commit sha does not match "
-                    "'{commit_sha}'."
+                    f"The '{desired_tag}' tag already exists, but the commit sha does not match "
+                    f"'{commit_sha}'."
                 )
 
     # Create a tag if one does not exist
@@ -138,7 +135,7 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path):
     loop = asyncio.get_event_loop()  # noqa
     # fmt: off
     package_found = asyncio.run(
-        get_package_from_pypi("pulp-file=={tag.name}", plugin_path)
+        get_package_from_pypi(f"pulp-file=={tag.name}", plugin_path)
     )  # noqa
     # fmt: on
     if not package_found: