diff --git a/.github/actions/build-server/action.yml b/.github/actions/build-server/action.yml new file mode 100644 index 00000000..7b9a77c6 --- /dev/null +++ b/.github/actions/build-server/action.yml @@ -0,0 +1,69 @@ +name: build-server +description: Sets up the build context and builds a docker image +inputs: + server-profile: + required: true + description: The profile of the server to build (discovery, registry, repository) + load: + required: false + description: Set to "true" to use image in further steps (only works for single architecture builds) + default: "false" + platform: + required: false + description: The target platforms to build the image for + default: "linux/amd64" + publish: + required: false + description: Set to "true" to publish image to Docker Hub (requires login) + default: "false" + +outputs: + image-ref: + description: Reference to the locally-runnable image (valid if load="true") + value: ${{ steps.naming.outputs.image-name }}:${{ steps.naming.outputs.local-tag }} + +runs: + using: composite + steps: + - name: Setup QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Generate Image Title + id: naming + shell: bash + run: | + profile="${{ inputs.server-profile }}" + echo "profile-title=${profile^}" >> "$GITHUB_OUTPUT" + echo "image-name=eclipsebasyx/basyx-python-${{ inputs.server-profile }}" >> $GITHUB_OUTPUT + echo "local-tag=ci-${{ github.run_id }}" >> $GITHUB_OUTPUT + + - name: Extract Docker image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ steps.naming.outputs.image-name }} + # This fetches the latest git tag and adds an additional "latest" to it, so e.g. `2.1.0,latest` + tags: | + type=semver,pattern={{version}} + type=raw,value=latest,enable=${{ inputs.publish == 'true' && true || false }} + type=raw,value=${{ steps.naming.outputs.local-tag }},enable=${{ inputs.load == 'true' && true || false }} + labels: | + org.opencontainers.image.title=BaSyx Python ${{ steps.naming.outputs.profile-title }} Service + org.opencontainers.image.description=Eclipse BaSyx Python SDK - ${{ steps.naming.outputs.profile-title }} HTTP Server + org.opencontainers.image.source=https://github.com/eclipse-basyx/basyx-python-sdk/tree/main/server + org.opencontainers.image.licenses=MIT + + - name: Build Docker Image + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ./server/docker/${{ inputs.server-profile }}/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: ${{ inputs.platform }} + push: ${{ inputs.publish == 'true' && true || false }} + load: ${{ inputs.load == 'true' && true || false }} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/pr.yml similarity index 62% rename from .github/workflows/ci.yml rename to .github/workflows/pr.yml index e71f1b5e..9bbdbbd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/pr.yml @@ -1,10 +1,21 @@ -name: ci +name: pr -on: [push, pull_request] +on: + push: + branches: [develop, main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: X_PYTHON_MIN_VERSION: "3.10" X_PYTHON_MAX_VERSION: "3.12" + X_API_VERSION: "v3.1" jobs: check-python-versions: @@ -14,11 +25,13 @@ jobs: run: working-directory: ./etc/scripts steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} + cache: "pip" + cache-dependency-path: "./check_python_versions_requirements.txt" - name: Install Python dependencies run: | python -m pip install --upgrade pip @@ -31,14 +44,23 @@ jobs: - name: Check Python Versions coincide with all pyproject.toml Files run: | - for file in ../../sdk/pyproject.toml ../../compliance_tool/pyproject.toml; do + for file in ../../sdk/pyproject.toml ../../compliance_tool/pyproject.toml ../../server/pyproject.toml; do python check_python_versions_coincide.py \ $file \ ${{ env.X_PYTHON_MIN_VERSION }} \ ${{ env.X_PYTHON_MAX_VERSION }} done - # Todo: Check other pyproject.toml here as well, as we add them + - name: Check Python Versions coincide with all Dockerfiles + run: | + for file in ../../server/docker/repository/Dockerfile \ + ../../server/docker/registry/Dockerfile \ + ../../server/docker/discovery/Dockerfile; do + python check_python_versions_coincide.py --docker \ + $file \ + ${{ env.X_PYTHON_MIN_VERSION }} \ + ${{ env.X_PYTHON_MAX_VERSION }} + done sdk-test: # This job runs the unittests on the python versions specified down at the matrix @@ -53,7 +75,7 @@ jobs: # that you want to use to test the serialization adapter against. # Currently, we need to update this manually, however I'm afraid this is not possible to automatically infer, # since it's heavily dependant of the version of the AAS specification we support. - AAS_SPECS_RELEASE_TAG: "IDTA-01001-3-1-2" + AAS_SPECS_RELEASE_TAG: "v3.1.2" services: couchdb: image: couchdb:3 @@ -66,7 +88,7 @@ jobs: run: working-directory: ./sdk steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Verify Matrix Version matches Global Version run: | if [ "${{ matrix.python-version }}" != "${{ env.X_PYTHON_MIN_VERSION }}" ] && [ "${{ matrix.python-version }}" != "${{ env.X_PYTHON_MAX_VERSION }}" ]; then @@ -74,14 +96,16 @@ jobs: exit 1 fi - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" - name: Collect schema files from aas-specs run: | mkdir -p ./test/adapter/schema - curl -sSL -o ./test/adapter/schema/aasJSONSchema.json https://raw.githubusercontent.com/admin-shell-io/aas-specs/${{ env.AAS_SPECS_RELEASE_TAG }}/schemas/json/aas.json - curl -sSL -o ./test/adapter/schema/aasXMLSchema.xsd https://raw.githubusercontent.com/admin-shell-io/aas-specs/${{ env.AAS_SPECS_RELEASE_TAG }}/schemas/xml/AAS.xsd + curl -sSLf -o ./test/adapter/schema/aasJSONSchema.json https://raw.githubusercontent.com/admin-shell-io/aas-specs-metamodel/refs/tags/${{ env.AAS_SPECS_RELEASE_TAG }}/schemas/json/aas.json + curl -sSLf -o ./test/adapter/schema/aasXMLSchema.xsd https://raw.githubusercontent.com/admin-shell-io/aas-specs-metamodel/refs/tags/${{ env.AAS_SPECS_RELEASE_TAG }}/schemas/xml/AAS.xsd - name: Install Python dependencies run: | python -m pip install --upgrade pip @@ -98,17 +122,19 @@ jobs: python -m coverage report -m sdk-static-analysis: - # This job runs static code analysis, namely pycodestyle and mypy + # This job runs static code analysis, namely ruff and mypy runs-on: ubuntu-latest defaults: run: working-directory: ./sdk steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" - name: Install Python dependencies run: | python -m pip install --upgrade pip @@ -116,28 +142,32 @@ jobs: - name: Check typing with MyPy run: | python -m mypy basyx test + - name: Check formatting and linting rules with Ruff + run: | + python -m ruff check sdk-readme-codeblocks: - # This job runs the same static code analysis (mypy and pycodestyle) on the codeblocks in our docstrings. + # This job runs the same static code analysis (mypy and ruff) on the codeblocks in our docstrings. runs-on: ubuntu-latest defaults: run: working-directory: ./sdk steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" - name: Install Python dependencies run: | python -m pip install --upgrade pip python -m pip install .[dev] - name: Check typing with MyPy - shell: bash run: | python -m mypy <(python -m codeblocks python README.md) - - name: Check code style with PyCodestyle + - name: Check formatting and linting rules with Ruff run: | python -m codeblocks --wrap python README.md | python -m pycodestyle --count --max-line-length 120 - - name: Run readme codeblocks with Python @@ -151,11 +181,13 @@ jobs: run: working-directory: ./sdk steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MAX_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MAX_VERSION }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" - name: Install Python dependencies run: | python -m pip install --upgrade pip @@ -171,9 +203,9 @@ jobs: run: working-directory: ./sdk steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} - name: Install dependencies @@ -188,13 +220,9 @@ jobs: # This job checks that the copyright year in the header of all files is up to date runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 with: fetch-depth: 0 # Ensure full history is available - - name: Install required dependencies - run: | - sudo apt-get update - sudo apt-get install -y bash git - name: Run copyright check run: | chmod +x ./etc/scripts/set_copyright_year.sh @@ -211,18 +239,15 @@ jobs: working-directory: ./compliance_tool steps: - - uses: actions/checkout@v4 - with: - # TODO(#592): revisit when the sdk<->compliance_tool version pinning is fixed. - # Need tags so setuptools_scm builds the local sdk as >=1.0.0, else the loose - # `basyx-python-sdk>=1.0.0` pin lets pip replace it with the PyPI release. - fetch-depth: 0 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" - name: Install Python dependencies - # install the local sdk first so its version satisfies the >=1.0.0 pin and pip keeps it instead of PyPI + # install the local sdk in editable mode so it does not get overwritten run: | python -m pip install --upgrade pip python -m pip install ../sdk @@ -236,25 +261,22 @@ jobs: python -m coverage report -m compliance-tool-static-analysis: - # This job runs static code analysis, namely pycodestyle and mypy + # This job runs static code analysis, namely ruff and mypy runs-on: ubuntu-latest defaults: run: working-directory: ./compliance_tool steps: - - uses: actions/checkout@v4 - with: - # TODO(#592): revisit when the sdk<->compliance_tool version pinning is fixed. - # Need tags so setuptools_scm builds the local sdk as >=1.0.0, else the loose - # `basyx-python-sdk>=1.0.0` pin lets pip replace it with the PyPI release. - fetch-depth: 0 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" - name: Install Python dependencies - # install the local sdk first so its version satisfies the >=1.0.0 pin and pip keeps it instead of PyPI + # install the local sdk in editable mode so it does not get overwritten run: | python -m pip install --upgrade pip python -m pip install ../sdk @@ -262,6 +284,9 @@ jobs: - name: Check typing with MyPy run: | python -m mypy aas_compliance_tool test + - name: Check formatting and linting rules with Ruff + run: | + python -m ruff check compliance-tool-package: # This job checks if we can build our compliance_tool package @@ -271,9 +296,9 @@ jobs: run: working-directory: ./compliance_tool steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} - name: Install dependencies @@ -284,24 +309,22 @@ jobs: run: | python -m build - #server-test: - # TODO: This job runs the unittests on the python versions specified down at the matrix - # and aas-test-engines on the server - server-static-analysis: - # This job runs static code analysis, namely pycodestyle and mypy + # This job runs static code analysis, namely ruff and mypy runs-on: ubuntu-latest defaults: run: working-directory: ./server steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: python-version: ${{ env.X_PYTHON_MIN_VERSION }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" - name: Install Python dependencies run: | python -m pip install --upgrade pip @@ -310,32 +333,42 @@ jobs: - name: Check typing with MyPy run: | python -m mypy app test + - name: Check formatting and linting rules with Ruff + run: | + python -m ruff check - server-repository-docker: + server-docker: # This job checks if we can build our server package runs-on: ubuntu-latest - defaults: - run: - working-directory: ./server/docker/repository + strategy: + matrix: + service: ["repository", "discovery", "registry"] + fail-fast: false steps: - - uses: actions/checkout@v4 - - name: Build the Docker image - run: | - docker build -t basyx-python-server -f Dockerfile ../../.. + - name: Checkout Repository + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + + - uses: ./.github/actions/build-server + id: build + with: + server-profile: ${{ matrix.service }} + load: true + publish: false + platform: linux/amd64 - name: Run container run: | - docker run -d --name basyx-python-server basyx-python-server + docker run -d --name basyx-python-${{ matrix.service }} -p 9080:80 --pull=never ${{ steps.build.outputs.image-ref }} - name: Wait for container and server initialization run: | timeout 30s bash -c ' - until docker logs basyx-python-server 2>&1 | grep -q "INFO success: quit_on_failure entered RUNNING state"; do + until docker logs basyx-python-${{ matrix.service }} 2>&1 | grep -q "INFO success: quit_on_failure entered RUNNING state"; do sleep 2 done ' - - name: Check if container is running + - name: Check if service is alive run: | - docker inspect --format='{{.State.Running}}' basyx-python-server | grep true + curl -f http://localhost:9080/api/${{ env.X_API_VERSION }}/description - name: Stop and remove the container run: | - docker stop basyx-python-server && docker rm basyx-python-server + docker stop basyx-python-${{ matrix.service }} && docker rm basyx-python-${{ matrix.service }} diff --git a/compliance_tool/pyproject.toml b/compliance_tool/pyproject.toml index 64f8adf8..a26b85a5 100644 --- a/compliance_tool/pyproject.toml +++ b/compliance_tool/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ dev = [ "mypy", "pycodestyle", + "ruff==0.16.0", "codeblocks", "coverage", ] diff --git a/etc/scripts/check_python_versions_coincide.py b/etc/scripts/check_python_versions_coincide.py index 013860d3..0bb4e3db 100644 --- a/etc/scripts/check_python_versions_coincide.py +++ b/etc/scripts/check_python_versions_coincide.py @@ -1,6 +1,6 @@ """ -This helper script checks if the Python versions defined in a `pyproject.toml` coincide with the given `min_version` -and `max_version` and returns an error if they don't. +This helper script checks if the Python versions defined in a `pyproject.toml` or `Dockerfile` coincide with the given +`min_version` and `max_version` and returns an error if they don't. """ import argparse import re @@ -8,41 +8,60 @@ from packaging.version import InvalidVersion, Version +def get_version_pyproject(file_path: str) -> str: + with open(file_path, "r") as f: + pyproject_content = f.read() -def main(pyproject_toml_path: str, min_version: str, max_version: str) -> None: + match = re.search(r'requires-python\s*=\s*">=([\d.]+)"', pyproject_content) + if not match: + print(f"Error: `requires-python` field not found or invalid format in `{file_path}`") + sys.exit(1) + + return match.group(1) + +def get_version_dockerfile(file_path: str) -> str: + with open(file_path, "r") as f: + pyproject_content = f.read() + + match = re.search(r'^FROM\s+python:([\d.]+)', pyproject_content) + if not match: + print(f"Error: Definition of base image `FROM python:x.x` not found in `{file_path}`") + sys.exit(1) + + return match.group(1) + +def main(file_path: str, is_dockerfile: bool, min_version: str, max_version: str) -> None: # Load and check `requires-python` version from `pyproject.toml` try: - with open(pyproject_toml_path, "r") as f: - pyproject_content = f.read() - - match = re.search(r'requires-python\s*=\s*">=([\d.]+)"', pyproject_content) - if not match: - print(f"Error: `requires-python` field not found or invalid format in `{pyproject_toml_path}`") - sys.exit(1) + if is_dockerfile: + used_version = get_version_dockerfile(file_path) + else: + used_version = get_version_pyproject(file_path) - pyproject_version = match.group(1) - if Version(pyproject_version) < Version(min_version): - print(f"Error: Python version in `{pyproject_toml_path}` `requires-python` ({pyproject_version}) " + if Version(used_version) < Version(min_version): + print(f"Error: Python version in `{file_path}` ({used_version}) " f"is smaller than `min_version` ({min_version}).") sys.exit(1) except FileNotFoundError: - print(f"Error: File not found: `{pyproject_toml_path}`.") + print(f"Error: File not found: `{file_path}`.") sys.exit(1) - print(f"Success: Version in pyproject.toml `requires-python` (>={pyproject_version}) " + print(f"Success: Version in `{file_path}` ({used_version}) " f"matches expected versions ([{min_version} to {max_version}]).") if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Check Python version support and alignment with pyproject.toml.") - parser.add_argument("pyproject_toml_path", help="Path to the `pyproject.toml` file to check.") + parser = argparse.ArgumentParser(description="Check Python version support and alignment with pyproject.toml or Dockerfile.") + parser.add_argument("file_path", help="Path to the `pyproject.toml` or `Dockerfile` file to check.") + parser.add_argument("--docker", action="store_true", + help="Set, if checking a `Dockerfile`, otherwise `pyproject.toml` is assumed.") parser.add_argument("min_version", help="The minimum Python version.") parser.add_argument("max_version", help="The maximum Python version.") args = parser.parse_args() try: - main(args.pyproject_toml_path, args.min_version, args.max_version) + main(args.file_path, args.docker, args.min_version, args.max_version) except InvalidVersion: print("Error: Invalid version format provided.") sys.exit(1) diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index b2a7a6b7..ba010aa4 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ dev = [ "mypy==1.15.0", "pycodestyle", + "ruff==0.16.0", "codeblocks", "coverage", "schemathesis~=3.7", diff --git a/server/pyproject.toml b/server/pyproject.toml index 8c40deff..e8799039 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ dev = [ "mypy", "pycodestyle", + "ruff==0.16.0", "codeblocks", "schemathesis~=3.7", "jsonschema~=4.7",