diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml new file mode 100644 index 0000000..f9035cb --- /dev/null +++ b/.github/workflows/assemble.yml @@ -0,0 +1,146 @@ +name: Assemble the bundle + +# The bundle's installers assembled from stagings, without compiling. The stagings are artifacts of this run, laid +# in by resolve.yml, -staging-. reusable.yml builds each platform as the stagings were built: +# the stagings unpacked each into its own folder, given to the bundle as FETCHLAYER_PREFIX_ (configure preset +# bundle-prefixes), the commits as BUNDLE_REVISION_, the installer packed (package preset installer). Each +# installer is then installed as a user would and smoke-checked by tests/installed. + +on: + workflow_call: + inputs: + revisions: + description: "The commit of each staging's layer: a JSON object, staging → full sha (resolve.yml's)" + type: string + platforms: + description: "The platforms: a JSON list of {job, triplet, generator, runner, modules} (resolve.yml's)" + type: string + +permissions: + contents: read + +jobs: + plan: + name: Plan the build + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.plan.outputs.jobs }} + packages: ${{ steps.plan.outputs.packages }} + cmake-presets: ${{ steps.plan.outputs.cmake-presets }} + cpack-presets: ${{ steps.plan.outputs.cpack-presets }} + checks: ${{ steps.plan.outputs.checks }} + steps: + - name: Plan + id: plan + shell: python3 {0} + env: + REVISIONS: ${{ inputs.revisions }} + PLATFORMS: ${{ inputs.platforms }} + run: | + import json, os + + # a staging → its layer, named as its repository (BUNDLE_REVISION_) + REPOSITORIES = {"opendaq": "openDAQ", "opcua-modules": "OpcUaModules", "mqtt-module": "MQTTStreamingModule", + "lt-streaming-modules": "LTStreamingModulesModern", "asam-cmp-modules": "ASAM-CMP-Modules"} + revision = json.loads(os.environ["REVISIONS"]) + platforms = json.loads(os.environ["PLATFORMS"]) + + packages, cmake_presets, cpack_presets, checks = [], [], [], [] + for p in platforms: + job = p["job"] + # the commits resolved there, so configure takes the same ones and resolves no ref again + artifacts = [] + cache_variables = {f"BUNDLE_REVISION_{REPOSITORIES[staging].upper()}": sha for staging, sha in revision.items()} + for staging in revision: + artifacts.append({"name": f"{staging}-staging-{job}", "dir": f"stagings/{staging}"}) + # the core's tarball carries its install prefix, opt/opendaq, except on Windows + inside = "/opt/opendaq" if staging == "opendaq" and not job.startswith("windows-") else "" + cache_variables[f"FETCHLAYER_PREFIX_{staging.upper()}"] = "${sourceDir}/stagings/" + staging + inside + packages.append({"match-jobs": [job], "artifacts": artifacts}) + cmake_presets.append({"match-jobs": [job], "configure-preset": "bundle-prefixes", "cache-variables": cache_variables}) + cpack_presets.append({"match-jobs": [job], "installers": [{"name": "bundle", "package": p["generator"], "package-preset": "installer"}]}) + checks.append({"job": job, "runner": p["runner"], "generator": p["generator"], "modules": p["modules"]}) + + # DEB names the libraries the stagings need in its Depends (dpkg-shlibdeps), so they must be there: ASAM's + # libpcap, ref_fb's renderer's X11, GL, udev and freetype, LT's OpenSSL; NSIS packs + packages += [{"match-jobs": ["ubuntu-*"], "apt-install": ["dpkg-dev", "file", "libpcap0.8", "libx11-6", + "libxrandr2", "libxcursor1", "libxi6", "libgl1", "libudev1", "libfreetype6", "libssl3"]}, + {"match-jobs": ["windows-*"], "choco-install": ["nsis"]}] + + outputs = {"jobs": [p["job"] for p in platforms], "packages": packages, + "cmake-presets": cmake_presets, "cpack-presets": cpack_presets, "checks": checks} + print(json.dumps(outputs, indent=2)) + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + for key, value in outputs.items(): + f.write(f"{key}={json.dumps(value)}\n") + + build: + name: Build + needs: plan + uses: openDAQ/openDAQ-CI/.github/workflows/reusable.yml@bundle/extract-each-artifact-into-its-dir + with: + include-jobs: ${{ needs.plan.outputs.jobs }} + packages: ${{ needs.plan.outputs.packages }} + cmake-presets: ${{ needs.plan.outputs.cmake-presets }} + cpack-presets: ${{ needs.plan.outputs.cpack-presets }} + + # installed as a user would, everything selected, and smoke-checked + check: + name: Check ${{ matrix.check.job }} + needs: [plan, build] + runs-on: ${{ matrix.check.runner }} + strategy: + fail-fast: false + matrix: + check: ${{ fromJSON(needs.plan.outputs.checks) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + + - uses: actions/download-artifact@v4 + with: + name: package-${{ matrix.check.job }} + path: installer + + - name: Install + id: install + timeout-minutes: 10 + run: | + find installer -type f + case "${{ matrix.check.generator }}" in + DEB) sudo apt-get update -qq + sudo apt-get install -y -qq "$(find "$PWD/installer" -name '*.deb' | head -1)" + echo "prefix=/opt/opendaq" >> "$GITHUB_OUTPUT" ;; + productbuild) sudo installer -pkg "$(find installer -name '*.pkg' | head -1)" -target / + echo "prefix=/opt/opendaq" >> "$GITHUB_OUTPUT" ;; + NSIS) exe=$(cygpath -w "$(find installer -name '*.exe' | head -1)") + pwsh -Command 'Write-Host "PATH: $($env:Path.Length) characters (the template reads 1024)"' + pwsh -Command "Start-Process -Wait -FilePath '$exe' -ArgumentList '/S','/PATH=AllUsers','/D=C:\opendaq'" + echo "prefix=C:/opendaq" >> "$GITHUB_OUTPUT" ;; + esac + + - name: Smoke check + run: | + cmake -S tests/installed -B build/installed \ + -DOPENDAQ_PREFIX=${{ steps.install.outputs.prefix }} -DEXPECT_MODULES=${{ matrix.check.modules }} + cmake --build build/installed --config Release + ctest --test-dir build/installed -C Release --output-on-failure + + # the installer added its bin to the machine's PATH, and the uninstaller removes it + - name: PATH + if: matrix.check.generator == 'NSIS' + timeout-minutes: 5 + shell: pwsh + run: | + $bin = 'C:\opendaq\bin' + $has = { ([Environment]::GetEnvironmentVariable('Path', 'Machine') -split ';') -contains $bin } + Write-Host "after install, machine PATH has ${bin}: $(& $has)" + if (-not (& $has)) { exit 1 } + Start-Process -Wait -FilePath 'C:\opendaq\Uninstall.exe' -ArgumentList '/S','_?=C:\opendaq' + Write-Host "after uninstall, machine PATH has ${bin}: $(& $has)" + if (& $has) { exit 1 } diff --git a/.github/workflows/bundle.yml b/.github/workflows/bundle.yml new file mode 100644 index 0000000..70ca0fb --- /dev/null +++ b/.github/workflows/bundle.yml @@ -0,0 +1,78 @@ +name: Bundle + +# The bundle's installers: resolve.yml lays the stagings of the bundle's commits into this run, assemble.yml builds +# the installers from them. A layer's revision can be given as an input (a commit, or a branch or tag taken at its +# head now), in place of bundle.json's. Called by the bundle's own triggers and by a layer's CI. + +on: + workflow_dispatch: + inputs: + opendaq: + description: "openDAQ: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + opcua-modules: + description: "OpcUaModules: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + mqtt-module: + description: "MQTTStreamingModule: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + lt-streaming-modules: + description: "LTStreamingModulesModern: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + asam-cmp-modules: + description: "ASAM-CMP-Modules: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + workflow_call: + inputs: + opendaq: + description: "openDAQ: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + opcua-modules: + description: "OpcUaModules: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + mqtt-module: + description: "MQTTStreamingModule: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + lt-streaming-modules: + description: "LTStreamingModulesModern: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + asam-cmp-modules: + description: "ASAM-CMP-Modules: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + pull_request: + +permissions: + contents: read + packages: read + +jobs: + resolve: + name: Resolve + permissions: + contents: read + packages: write # a missing staging's stage.yml asks for it; nothing is pushed from here + uses: ./.github/workflows/resolve.yml + with: + opendaq: ${{ inputs.opendaq }} + opcua-modules: ${{ inputs.opcua-modules }} + mqtt-module: ${{ inputs.mqtt-module }} + lt-streaming-modules: ${{ inputs.lt-streaming-modules }} + asam-cmp-modules: ${{ inputs.asam-cmp-modules }} + + assemble: + name: Assemble + needs: resolve + uses: ./.github/workflows/assemble.yml + with: + revisions: ${{ needs.resolve.outputs.revisions }} + platforms: ${{ needs.resolve.outputs.platforms }} diff --git a/.github/workflows/resolve.yml b/.github/workflows/resolve.yml new file mode 100644 index 0000000..98ee2ae --- /dev/null +++ b/.github/workflows/resolve.yml @@ -0,0 +1,297 @@ +name: Resolve the bundle + +# What the bundle is assembled of: a commit per layer and the platforms, and every staging laid into this run as an +# artifact, -staging-, as a layer's stage names it, for assemble.yml to build from. The commits are +# bundle.json's, each replaced by an input that is set (a commit, or a branch or tag taken at its head now). The +# stagings come from the registry, by staging-pull, each from the channel its commit lies in (openDAQ-CI's +# resolve-channel: release/** stable, main dev, dev also for a commit on neither, staged by hand); a module's is asked +# for by the core it was built against too, so the modules of one bundle share its core. A staging the registry lacks +# is staged here by its layer's stage.yml, called, never pushed; a module after the core is in the run, which its +# stage then does not pull again. + +on: + workflow_call: + inputs: + opendaq: + description: "openDAQ: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + opcua-modules: + description: "OpcUaModules: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + mqtt-module: + description: "MQTTStreamingModule: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + lt-streaming-modules: + description: "LTStreamingModulesModern: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + asam-cmp-modules: + description: "ASAM-CMP-Modules: a commit, branch or tag (empty: bundle.json's)" + type: string + default: "" + outputs: + revisions: + description: "The commit of each staging's layer: a JSON object, staging → full sha" + value: ${{ jobs.resolve.outputs.revisions }} + platforms: + description: "The platforms: a JSON list of {job, triplet, generator, runner, modules}" + value: ${{ jobs.resolve.outputs.platforms }} + +permissions: + contents: read + packages: read + +jobs: + resolve: + name: Resolve the stagings + runs-on: ubuntu-latest + outputs: + revisions: ${{ steps.stagings.outputs.revisions }} + platforms: ${{ steps.stagings.outputs.platforms }} + exclude-jobs: ${{ steps.stagings.outputs.exclude-jobs }} + staging-artifacts: ${{ steps.split.outputs.staging-artifacts }} + missing: ${{ steps.split.outputs.missing }} + steps: + # bundle.json of this workflow's own commit, whoever calls it + - uses: actions/checkout@v4 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + + - name: Resolve the commits + id: stagings + shell: python3 {0} + env: + INPUTS: ${{ toJSON(inputs) }} + run: | + import json, os, re, subprocess, sys + + # one entry per platform the bundle is assembled for: + # key the reusable.yml job that packs it (its name picks the runner, the compiler, the generator) + # triplet the stagings it takes: com.opendaq.package.triplet of the packages pulled from ghcr + # generator the CPack generator of its installer + # runner where the installer is installed and smoke-checked (tests/installed) + # modules how many modules the smoke check expects to load (Windows: ASAM's two need Npcap, absent) + # runner and modules serve the smoke check, a debugging aid until the integration tests take the installers; + # they go with it + JOBS = { + # job triplet generator runner modules + "ubuntu-22.04-x86_64-ninja-gcc-release": ("x86_64-linux-gcc-11-release", "DEB", "ubuntu-22.04", 14), + "ubuntu-22.04-armv8-ninja-gcc-release": ("armv8-linux-gcc-11-release", "DEB", "ubuntu-24.04-arm", 14), + "windows-2022-x86_64-msvs-v143-release": ("x86_64-windows-msvc-143-release", "NSIS", "windows-2022", 12), + "macos-26-armv8-ninja-appleclang-21-release": ("armv8-macos-apple-clang-21-release", "productbuild", "macos-26", 14), + "macos-26-x86_64-ninja-appleclang-21-release": ("x86_64-macos-apple-clang-21-release", "productbuild", "macos-26-intel", 14), + } + # the jobs of a layer's stage the bundle takes nothing from: not built when a layer is staged here + EXCLUDE_JOBS = ["manylinux*", "windows-*-x86-*"] + # a layer, named as its repository in bundle.json → its staging + STAGINGS = {"openDAQ": "opendaq", "OpcUaModules": "opcua-modules", "MQTTStreamingModule": "mqtt-module", + "LTStreamingModulesModern": "lt-streaming-modules", "ASAM-CMP-Modules": "asam-cmp-modules"} + REPOSITORIES = {staging: layer for layer, staging in STAGINGS.items()} + + inputs = json.loads(os.environ["INPUTS"] or "{}") or {} + + def commit(staging, ref): + """A full sha as it is; a branch or a tag → its commit, as the repository has it now.""" + if re.fullmatch(r"[0-9a-f]{40}", ref): + return ref + url = f"https://github.com/openDAQ/{REPOSITORIES[staging]}.git" + out = subprocess.run(["git", "ls-remote", url, ref], capture_output=True, text=True, check=True).stdout + refs = dict(reversed(line.split("\t")) for line in out.splitlines()) + for name in (f"refs/tags/{ref}^{{}}", f"refs/tags/{ref}", f"refs/heads/{ref}"): + if name in refs: + return refs[name] + sys.exit(f"::error::{REPOSITORIES[staging]} has no branch or tag {ref!r} (a commit is given in full)") + + # bundle.json's revisions, each replaced by an input that is set + revision = {} + for layer, ref in json.load(open("bundle.json")).items(): + staging = STAGINGS[layer] + given = (inputs.get(staging) or "").strip() + revision[staging] = commit(staging, given or ref) + + summary = ["| Staging | Revision |", "|-|-|"] + for staging, sha in revision.items(): + summary.append(f"| {staging} | [`{sha[:12]}`](https://github.com/openDAQ/{REPOSITORIES[staging]}/commit/{sha}) |") + print("\n".join(summary)) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write("\n".join(summary) + "\n") + + platforms = [{"job": job, "triplet": triplet, "generator": generator, "runner": runner, "modules": modules} + for job, (triplet, generator, runner, modules) in JOBS.items()] + # the modules' commits, by repository, for their channels + modules = {f"openDAQ/{REPOSITORIES[staging]}": sha for staging, sha in revision.items() if staging != "opendaq"} + + outputs = {"revisions": revision, "platforms": platforms, "modules": modules, "exclude-jobs": EXCLUDE_JOBS, + "repositories": {staging: f"openDAQ/{layer}" for staging, layer in REPOSITORIES.items()}} + print(json.dumps(outputs, indent=2)) + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + for key, value in outputs.items(): + f.write(f"{key}={json.dumps(value)}\n") + + # the channel each staging is published in, by where its commit lies: the core's commit given as the commit, + # the modules' as what it is built against, only to learn theirs (nothing is published here) + - name: Resolve the channels + id: channel + uses: openDAQ/openDAQ-CI/.github/actions/resolve-channel@bundle/extract-each-artifact-into-its-dir + with: + repository: openDAQ/openDAQ + commit: ${{ fromJSON(steps.stagings.outputs.revisions)['opendaq'] }} + depends: ${{ steps.stagings.outputs.modules }} + + - name: Request the stagings + id: request + shell: python3 {0} + env: + REVISIONS: ${{ steps.stagings.outputs.revisions }} + PLATFORMS: ${{ steps.stagings.outputs.platforms }} + REPOSITORIES: ${{ steps.stagings.outputs.repositories }} + CORE_CHANNEL: ${{ steps.channel.outputs.channel }} + MODULE_CHANNELS: ${{ steps.channel.outputs.depends-channels }} + run: | + import json, os + + revision = json.loads(os.environ["REVISIONS"]) + repository = json.loads(os.environ["REPOSITORIES"]) + # a staging is looked for in its commit's channel, and in dev when it lies on no main or release/** (staged + # by hand) + channels = {"openDAQ/openDAQ": os.environ["CORE_CHANNEL"], **json.loads(os.environ["MODULE_CHANNELS"])} + channel = {staging: channels[repository[staging]] or "dev" for staging in revision} + + # every staging of every platform, by its triplet, into an artifact named as the staging's own stage names + # it; optional: the check tells what is there + pull = [] + for platform in json.loads(os.environ["PLATFORMS"]): + job, triplet = platform["job"], platform["triplet"] + for staging, sha in revision.items(): + annotations = {"org.opencontainers.image.revision": sha, "com.opendaq.package.triplet": triplet} + if staging != "opendaq": + annotations["com.opendaq.dependencies.opendaq.revision"] = revision["opendaq"] + pull.append({"artifact": f"{staging}-staging-{job}", + "packages": [{"path": f"ghcr.io/openDAQ/{channel[staging]}/staging/{staging}", + "annotations": annotations, "optional": True}]}) + print(json.dumps(channel, indent=2)) + + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"request={json.dumps(pull)}\n") + + - name: Check the registry + id: check + uses: openDAQ/openDAQ-CI/.github/actions/staging-check@v4 + with: + packages: ${{ steps.request.outputs.request }} + + # a staging missing on any platform is staged here, on all of them; the rest is pulled + - name: Split + id: split + shell: python3 {0} + env: + REQUEST: ${{ steps.request.outputs.request }} + RESOLVED: ${{ steps.check.outputs.resolved }} + run: | + import json, os + + request = json.loads(os.environ["REQUEST"]) + found = {entry["artifact"] for entry in json.loads(os.environ["RESOLVED"]) if entry["packages"]} + staging_of = lambda entry: entry["packages"][0]["path"].rsplit("/", 1)[1] + missing = sorted({staging_of(entry) for entry in request if entry["artifact"] not in found}) + + pull = [{**entry, "packages": [{k: v for k, v in package.items() if k != "optional"} for package in entry["packages"]]} + for entry in request if staging_of(entry) not in missing] + print(f"missing, staged here: {missing or 'none'}") + + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"staging-artifacts={json.dumps(pull)}\n") + f.write(f"missing={json.dumps(missing)}\n") + + pull: + name: Pull + needs: resolve + if: needs.resolve.outputs.staging-artifacts != '[]' + uses: openDAQ/openDAQ-CI/.github/workflows/staging-pull.yml@v4 + with: + staging-artifacts: ${{ needs.resolve.outputs.staging-artifacts }} + + stage-opendaq: + name: Stage openDAQ + needs: resolve + if: contains(fromJSON(needs.resolve.outputs.missing), 'opendaq') + permissions: + contents: read + packages: write # asked for by the stage.yml's push, which is never run from here + uses: openDAQ/openDAQ/.github/workflows/stage.yml@bundle/stagings + with: + ref: ${{ fromJSON(needs.resolve.outputs.revisions)['opendaq'] }} + exclude-jobs: ${{ needs.resolve.outputs.exclude-jobs }} + + # after the core is in the run, pulled or staged, so the module's stage does not pull it again + stage-opcua-modules: + name: Stage OpcUaModules + needs: [resolve, pull, stage-opendaq] + if: >- + always() && needs.resolve.result == 'success' + && !contains(fromJSON('["failure", "cancelled"]'), needs.pull.result) + && !contains(fromJSON('["failure", "cancelled"]'), needs.stage-opendaq.result) + && contains(fromJSON(needs.resolve.outputs.missing), 'opcua-modules') + permissions: + contents: read + packages: write # asked for by the stage.yml's push, which is never run from here + uses: openDAQ/OpcUaModules/.github/workflows/stage.yml@bundle/stagings + with: + ref: ${{ fromJSON(needs.resolve.outputs.revisions)['opcua-modules'] }} + opendaq-ref: ${{ fromJSON(needs.resolve.outputs.revisions)['opendaq'] }} + exclude-jobs: ${{ needs.resolve.outputs.exclude-jobs }} + + stage-mqtt-module: + name: Stage MQTTStreamingModule + needs: [resolve, pull, stage-opendaq] + if: >- + always() && needs.resolve.result == 'success' + && !contains(fromJSON('["failure", "cancelled"]'), needs.pull.result) + && !contains(fromJSON('["failure", "cancelled"]'), needs.stage-opendaq.result) + && contains(fromJSON(needs.resolve.outputs.missing), 'mqtt-module') + permissions: + contents: read + packages: write # asked for by the stage.yml's push, which is never run from here + uses: openDAQ/MQTTStreamingModule/.github/workflows/stage.yml@bundle/stagings + with: + ref: ${{ fromJSON(needs.resolve.outputs.revisions)['mqtt-module'] }} + opendaq-ref: ${{ fromJSON(needs.resolve.outputs.revisions)['opendaq'] }} + exclude-jobs: ${{ needs.resolve.outputs.exclude-jobs }} + + stage-lt-streaming-modules: + name: Stage LTStreamingModulesModern + needs: [resolve, pull, stage-opendaq] + if: >- + always() && needs.resolve.result == 'success' + && !contains(fromJSON('["failure", "cancelled"]'), needs.pull.result) + && !contains(fromJSON('["failure", "cancelled"]'), needs.stage-opendaq.result) + && contains(fromJSON(needs.resolve.outputs.missing), 'lt-streaming-modules') + permissions: + contents: read + packages: write # asked for by the stage.yml's push, which is never run from here + uses: openDAQ/LTStreamingModulesModern/.github/workflows/stage.yml@bundle/stagings + with: + ref: ${{ fromJSON(needs.resolve.outputs.revisions)['lt-streaming-modules'] }} + opendaq-ref: ${{ fromJSON(needs.resolve.outputs.revisions)['opendaq'] }} + exclude-jobs: ${{ needs.resolve.outputs.exclude-jobs }} + + stage-asam-cmp-modules: + name: Stage ASAM-CMP-Modules + needs: [resolve, pull, stage-opendaq] + if: >- + always() && needs.resolve.result == 'success' + && !contains(fromJSON('["failure", "cancelled"]'), needs.pull.result) + && !contains(fromJSON('["failure", "cancelled"]'), needs.stage-opendaq.result) + && contains(fromJSON(needs.resolve.outputs.missing), 'asam-cmp-modules') + permissions: + contents: read + packages: write # asked for by the stage.yml's push, which is never run from here + uses: openDAQ/ASAM-CMP-Modules/.github/workflows/stage.yml@bundle/stagings + with: + ref: ${{ fromJSON(needs.resolve.outputs.revisions)['asam-cmp-modules'] }} + opendaq-ref: ${{ fromJSON(needs.resolve.outputs.revisions)['opendaq'] }} + exclude-jobs: ${{ needs.resolve.outputs.exclude-jobs }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d16b94..0b4f967 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,162 +1,32 @@ set(CMAKE_POLICY_VERSION_MINIMUM 3.5) cmake_minimum_required(VERSION 3.25) -set(REPO_NAME ReleaseRepo) -set(REPO_OPTION_PREFIX RELEASE_REPO) -set(CMAKE_FOLDER "${CMAKE_FOLDER}/${REPO_NAME}") +project(openDAQ-bundle VERSION 1.0.0 LANGUAGES CXX) - -list(APPEND CMAKE_MESSAGE_CONTEXT ${REPO_NAME}) - -project(${REPO_NAME} VERSION "1.0.0" LANGUAGES CXX) - -option(${REPO_OPTION_PREFIX}_ENABLE_TESTS "Enable testing" OFF) -option(${REPO_OPTION_PREFIX}_ENABLE_OPCUA_MODULE "Enable OPC UA module" ON) -option(${REPO_OPTION_PREFIX}_ENABLE_MQTT_MODULE "Enable MQTT module" ON) -option(${REPO_OPTION_PREFIX}_ENABLE_LT_MODULE "Enable LT streaming module" ON) -option(${REPO_OPTION_PREFIX}_ENABLE_ASAM_CMP_MODULE "Enable ASAM-CMP modules" ON) +# The bundle: the stagings named, each from where the policy says — prebuilt from the store, or built from its +# layer's sources (cmake/FetchLayer.cmake). What the bundle is made of is cmake/Layers.cmake; how it is packed, +# cmake/Packing.cmake. +set(BUNDLE_STAGINGS "opendaq;opcua-modules;mqtt-module;lt-streaming-modules;asam-cmp-modules" + CACHE STRING "The stagings to make available, dependencies first") +option(BUNDLE_ENABLE_TESTS "Build the layers' tests, for the layers built from sources" OFF) include(FetchContent) -# fetch the cmake utilities first, as they are used by the other modules +# the cmake utilities first: the triplet, Boost, and what the layers built here use; the tag the core takes FetchContent_Declare( opendaq-cmake-utils GIT_REPOSITORY https://github.com/openDAQ/opendaq-cmake-utils.git - GIT_TAG v1.0.1 + GIT_TAG v1.1.3 GIT_PROGRESS ON ) FetchContent_MakeAvailable(opendaq-cmake-utils) -FetchContent_Declare( - opendaq - GIT_REPOSITORY https://github.com/openDAQ/openDAQ.git - GIT_TAG main - GIT_PROGRESS ON - GIT_SHALLOW OFF - SOURCE_SUBDIR "INVALID" - GIT_REMOTE_UPDATE_STRATEGY CHECKOUT -) - -set(OPENDAQ_ENABLE_OPCUA OFF CACHE BOOL "") -set(OPENDAQ_ENABLE_OPCUA_INTEGRATION_TESTS OFF CACHE BOOL "") - -set(OPENDAQ_GENERATE_DELPHI_BINDINGS OFF CACHE BOOL "" FORCE) -set(OPENDAQ_GENERATE_PYTHON_BINDINGS ON CACHE BOOL "") -set(OPENDAQ_GENERATE_CSHARP_BINDINGS ON CACHE BOOL "") -set(OPENDAQ_GENERATE_C_BINDINGS ON CACHE BOOL "") - -set(OPENDAQ_ENABLE_TESTS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(OPENDAQ_ENABLE_TEST_UTILS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(OPENDAQ_ENABLE_OPTIONAL_TESTS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(OPENDAQ_ENABLE_COVERAGE OFF CACHE BOOL "" FORCE) -set(OPENDAQ_ENABLE_REGRESSION_TESTS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(OPENDAQ_ENABLE_UNSTABLE_TEST_LABELS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) - -set(DAQMODULES_EMPTY_MODULE ON CACHE BOOL "") -set(DAQMODULES_OPENDAQ_CLIENT_MODULE ON CACHE BOOL "") -set(DAQMODULES_OPENDAQ_SERVER_MODULE ON CACHE BOOL "") -set(DAQMODULES_REF_DEVICE_MODULE ON CACHE BOOL "") -set(DAQMODULES_SIMULATOR_DEVICE_MODULE ON CACHE BOOL "") -set(DAQMODULES_REF_FB_MODULE ON CACHE BOOL "") -set(DAQMODULES_AUDIO_DEVICE_MODULE ON CACHE BOOL "") -set(DAQMODULES_BASIC_CSV_RECORDER_MODULE ON CACHE BOOL "") -set(DAQMODULES_LICENSING_MODULE ON CACHE BOOL "") -set(DAQMODULES_PARQUET_RECORDER_MODULE OFF CACHE BOOL "") - -set(OPENDAQ_ENABLE_NATIVE_STREAMING ON CACHE BOOL "") -set(OPENDAQ_ENABLE_WEBSOCKET_STREAMING OFF CACHE BOOL "") - -set(OPENDAQ_DOCUMENTATION_TESTS OFF CACHE BOOL "" FORCE) -set(OPENDAQ_ENABLE_PYTHON_BINDINGS_TESTS OFF CACHE BOOL "" FORCE) -set(APP_ENABLE_WEBPAGE_EXAMPLES OFF CACHE BOOL "") - -FetchContent_MakeAvailable(opendaq) -include("${opendaq_SOURCE_DIR}/external/boost/Boost.cmake" OPTIONAL) - -if (${REPO_OPTION_PREFIX}_ENABLE_OPCUA_MODULE) -FetchContent_Declare( - opcuamodules - GIT_REPOSITORY https://github.com/openDAQ/OpcUaModules.git - GIT_TAG main - GIT_PROGRESS ON - GIT_SHALLOW OFF - SOURCE_SUBDIR "INVALID" - GIT_REMOTE_UPDATE_STRATEGY CHECKOUT -) - -set(DAQMODULES_OPCUA_ENABLE_EXAMPLE_APP OFF CACHE BOOL "") -set(DAQMODULES_OPCUA_ENABLE_TESTS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(DAQMODULES_OPCUA_ENABLE_CLIENT ON CACHE BOOL "") -set(DAQMODULES_OPCUA_ENABLE_SERVER ON CACHE BOOL "") -set(DAQMODULES_OPCUA_ENABLE_GENERIC_CLIENT ON CACHE BOOL "") - -FetchContent_MakeAvailable(opcuamodules) -include("${opcuamodules_SOURCE_DIR}/external/boost/Boost.cmake" OPTIONAL) -endif() - -if (${REPO_OPTION_PREFIX}_ENABLE_MQTT_MODULE) -FetchContent_Declare( - mqttmodule - GIT_REPOSITORY https://github.com/openDAQ/MQTTStreamingModule.git - GIT_TAG main - GIT_PROGRESS ON - GIT_SHALLOW OFF - SOURCE_SUBDIR "INVALID" - GIT_REMOTE_UPDATE_STRATEGY CHECKOUT -) - -set(DAQMODULES_MQTT_ENABLE_TESTS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(DAQMODULES_MQTT_ENABLE_EXAMPLE_APP OFF CACHE BOOL "") -set(DAQMODULES_MQTT_ENABLE_SSL OFF CACHE BOOL "") - -FetchContent_MakeAvailable(mqttmodule) -include("${mqttmodule_SOURCE_DIR}/external/boost/Boost.cmake" OPTIONAL) -endif() - - -if (${REPO_OPTION_PREFIX}_ENABLE_LT_MODULE) -FetchContent_Declare( - ltstreamingmodulesmodern - GIT_REPOSITORY https://github.com/openDAQ/LTStreamingModulesModern.git - GIT_TAG main - GIT_PROGRESS ON - GIT_SHALLOW OFF - SOURCE_SUBDIR "INVALID" - GIT_REMOTE_UPDATE_STRATEGY CHECKOUT -) - -set(DAQMODULES_LT_STREAMING_ENABLE_TESTS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(DAQMODULES_LT_STREAMING_ENABLE_EXAMPLE_APP OFF CACHE BOOL "") -set(DAQMODULES_LT_STREAMING_ENABLE_CLIENT ON CACHE BOOL "") -set(DAQMODULES_LT_STREAMING_ENABLE_SERVER ON CACHE BOOL "") - -FetchContent_MakeAvailable(ltstreamingmodulesmodern) -include("${ltstreamingmodulesmodern_SOURCE_DIR}/external/boost/Boost.cmake" OPTIONAL) -endif() - -if (${REPO_OPTION_PREFIX}_ENABLE_ASAM_CMP_MODULE) -FetchContent_Declare( - asamcmpmodules - GIT_REPOSITORY https://github.com/openDAQ/ASAM-CMP-Modules.git - GIT_TAG main - GIT_PROGRESS ON - GIT_SHALLOW OFF - SOURCE_SUBDIR "INVALID" - GIT_REMOTE_UPDATE_STRATEGY CHECKOUT -) - -set(ASAM_CMP_ENABLE_TESTS ${${REPO_OPTION_PREFIX}_ENABLE_TESTS} CACHE BOOL "" FORCE) -set(ASAM_CMP_ENABLE_EXAMPLE OFF CACHE BOOL "") -set(ASAM_CMP_BUILD_DATA_SINK ON CACHE BOOL "") -set(ASAM_CMP_BUILD_CAPTURE_MODULE ON CACHE BOOL "") - -FetchContent_MakeAvailable(asamcmpmodules) -include("${asamcmpmodules_SOURCE_DIR}/external/boost/Boost.cmake" OPTIONAL) -endif() +include(cmake/FetchLayer.cmake) +# the identity keys are the annotations staging-push writes +set(FETCHLAYER_IDENTITY_REVISION_KEY "org.opencontainers.image.revision") +set(FETCHLAYER_IDENTITY_DEPENDENCY_KEY "com.opendaq.dependencies..revision") +set(FETCHLAYER_IDENTITY_AXIS_KEY "com.opendaq.package.") -# add the subdirectories for the core and modules -add_subdirectory(${opendaq_SOURCE_DIR} ${opendaq_BINARY_DIR}) -add_subdirectory(${opcuamodules_SOURCE_DIR} ${opcuamodules_BINARY_DIR}) -add_subdirectory(${mqttmodule_SOURCE_DIR} ${mqttmodule_BINARY_DIR}) -add_subdirectory(${ltstreamingmodulesmodern_SOURCE_DIR} ${ltstreamingmodulesmodern_BINARY_DIR}) -add_subdirectory(${asamcmpmodules_SOURCE_DIR} ${asamcmpmodules_BINARY_DIR}) +include(cmake/Layers.cmake) +FetchLayer_MakeAvailable(${BUNDLE_STAGINGS}) +include(cmake/Packing.cmake) diff --git a/CMakePresets.json b/CMakePresets.json index ac83377..2ddeda7 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,5 +1,5 @@ { - "version": 4, + "version": 6, "configurePresets": [ { "name": "debug", @@ -100,6 +100,29 @@ "msvc-x86" ] }, + { + "name": "bundle-prefixes", + "displayName": "Bundle from stagings given as prefixes (FETCHLAYER_PREFIX_), nothing compiled; CI", + "inherits": [ + "release" + ], + "cacheVariables": { + "BUNDLE_STAGINGS": "opendaq;opcua-modules;mqtt-module;lt-streaming-modules;asam-cmp-modules", + "FETCHLAYER_BUILD": "NEVER" + } + }, + { + "name": "bundle-native", + "displayName": "Bundle by FetchLayer, this machine's triplet", + "inherits": [ + "release", + "ninja" + ], + "cacheVariables": { + "BUNDLE_STAGINGS": "opendaq;opcua-modules;mqtt-module;lt-streaming-modules;asam-cmp-modules", + "FETCHLAYER_STORE": "${sourceDir}/build/store" + } + }, { "name": "x64/msvc-17", "displayName": "[MSVC 17] Release - x64", @@ -163,5 +186,49 @@ "ninja" ] } + ], + "buildPresets": [ + { + "name": "bundle-native", + "configurePreset": "bundle-native" + } + ], + "packagePresets": [ + { + "name": "core", + "displayName": "The core: runtime, C++ API, C", + "configurePreset": "bundle-native", + "generators": [ + "TGZ" + ], + "variables": { + "BUNDLE_PACK_STAGINGS": "opendaq", + "BUNDLE_PACK_COMPONENTS": "Runtime;Development", + "CPACK_OPENDAQ_META_PACKAGE_NAME": "opendaq-core" + } + }, + { + "name": "installer", + "displayName": "The core with the modules, the modules optional where the generator offers choices (-G DEB, NSIS, productbuild)", + "configurePreset": "bundle-native", + "variables": { + "BUNDLE_PACK_STAGINGS": "opendaq;opcua-modules;mqtt-module;lt-streaming-modules;asam-cmp-modules", + "BUNDLE_PACK_COMPONENTS": "Runtime;Development", + "CPACK_OPENDAQ_META_PACKAGE_NAME": "opendaq-bundle" + } + }, + { + "name": "modules", + "displayName": "The modules, without the core", + "configurePreset": "bundle-native", + "generators": [ + "TGZ" + ], + "variables": { + "BUNDLE_PACK_STAGINGS": "opcua-modules;mqtt-module;lt-streaming-modules;asam-cmp-modules", + "BUNDLE_PACK_COMPONENTS": "Runtime", + "CPACK_OPENDAQ_META_PACKAGE_NAME": "opendaq-modules" + } + } ] } diff --git a/README.md b/README.md index 93b91f8..5a596a4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,11 @@ A CMake super-project that bundles [openDAQ](https://github.com/openDAQ/openDAQ) - [LTStreamingModulesModern](https://github.com/openDAQ/LTStreamingModulesModern) - [ASAM-CMP-Modules](https://github.com/openDAQ/ASAM-CMP-Modules) -All dependencies are fetched automatically via CMake's `FetchContent`. +Each of these repositories is a **layer** at one revision. What a layer publishes is its **staging**: `opendaq`, +`opcua-modules`, `mqtt-module`, `lt-streaming-modules`, `asam-cmp-modules`. The bundle takes each staging prebuilt, +from a store of stagings, when one fits its revision, its dependencies' revisions and this platform. Otherwise it +builds the layer from its sources in the same tree, fetched with CMake's `FetchContent`. The engine is +`cmake/FetchLayer.cmake`; what the bundle is made of is `cmake/Layers.cmake`. ## Prerequisites @@ -14,6 +18,12 @@ All dependencies are fetched automatically via CMake's `FetchContent`. - Git - **Windows:** Visual Studio 2017 or 2022 (with C++ workload) — used as compiler toolchain only - **Linux:** GCC or Clang, Ninja +- **macOS:** Xcode command line tools, Ninja +- For prebuilt stagings: [oras](https://oras.land) and read access to `ghcr.io/opendaq/dev/staging` + +On an Intel Mac with Homebrew in `/usr/local`, Homebrew's `boost`, `spdlog`, `fmt` and `rapidjson` headers shadow +the ones the build fetches. Unlink them for a build from sources (`brew unlink boost spdlog fmt rapidjson`, then +`brew link …`). ## Configure @@ -21,6 +31,7 @@ Use one of the available CMake presets: | Preset | Platform | Compiler | Arch | Config | |---|---|---|---|---| +| `bundle-native` | this machine | default | this machine | Release, prebuilt stagings from `build/store` | | `x64/msvc-22` | Windows | MSVC 2022 | x64 | Release | | `x86/msvc-22` | Windows | MSVC 2022 | x86 | Release | | `x64/msvc-17` | Windows | MSVC 2017 | x64 | Release | @@ -30,6 +41,9 @@ Use one of the available CMake presets: | `x64/clang/release` | Linux | Clang | x64 | Release | | `x64/clang/debug` | Linux | Clang | x64 | Debug | +`bundle-native` pins every layer to a commit whose stagings are published. The others set no store, so they build +every layer from its `main` branch. + ```bash # Windows cmake --preset="x64/msvc-22" @@ -52,14 +66,56 @@ cmake --build build/x64/gcc/release | Option | Default | Description | |---|---|---| -| `RELEASE_REPO_ENABLE_TESTS` | `OFF` | Build and enable tests | -| `RELEASE_REPO_ENABLE_OPCUA_MODULE` | `ON` | Include OPC UA module | -| `RELEASE_REPO_ENABLE_MQTT_MODULE` | `ON` | Include MQTT streaming module | -| `RELEASE_REPO_ENABLE_LT_MODULE` | `ON` | Include LT streaming module | -| `RELEASE_REPO_ENABLE_ASAM_CMP_MODULE` | `ON` | Include ASAM-CMP modules | +| `BUNDLE_STAGINGS` | all five | The stagings to make available, dependencies first: `opendaq;opcua-modules;mqtt-module;lt-streaming-modules;asam-cmp-modules` | +| `BUNDLE_REVISION_` | `main` | The revision of a layer (a repository, its name upper-cased): a full sha, a tag or a branch, e.g. `BUNDLE_REVISION_OPCUAMODULES` | +| `FETCHLAYER_BUILD` | `MISSING` | `NEVER`: prebuilt stagings only; `MISSING`: prebuilt where there is one, else from sources; `ALWAYS`: from sources. `FETCHLAYER_BUILD_` for one layer | +| `FETCHLAYER_STORE` | — | The store of prebuilt stagings, `//idx-//`; without one, everything is built from sources | +| `FETCHCONTENT_SOURCE_DIR_` | — | A layer from a local checkout, always built from it | +| `BUNDLE_AXIS_TRIPLET` | this build's | The triplets to make available; others than this build's come prebuilt, as files only | +| `BUNDLE_ENABLE_TESTS` | `OFF` | Build the tests of the layers built from sources | Pass options at configure time: ```bash -cmake --preset="x64/gcc/release" -DRELEASE_REPO_ENABLE_TESTS=ON +cmake --preset="x64/gcc/release" -DBUNDLE_ENABLE_TESTS=ON +cmake --preset="x64/gcc/release" -DBUNDLE_STAGINGS="opendaq;opcua-modules" -DBUNDLE_REVISION_OPCUAMODULES=v4.0.1 +cmake --preset bundle-native -DFETCHLAYER_BUILD_OPCUAMODULES=ALWAYS # OPC UA from sources, the rest prebuilt +``` + +## Prebuilt stagings + +The store is filled from the registry once, for the revisions `bundle-native` pins (see `CMakePresets.json`): + +```bash +cmake -DSTORE=build/store -DREGISTRY=ghcr.io/opendaq/dev/staging \ + [-DORAS_REGISTRY_CONFIG=$HOME/.oras/config.json] \ + "-DLAYERS=opendaq=;opcua-modules=;mqtt-module=;lt-streaming-modules=;asam-cmp-modules=" \ + -P tools/fill-store.cmake +``` + +It takes every platform of each staging; a build already in the store is kept. Configuring from a filled store +needs no network, apart from the first fetch of `opendaq-cmake-utils`. + +## Package + +With prebuilt stagings in a store, `bundle-native` configures without building, and the package presets pack: + +```bash +cmake --preset bundle-native +cmake --build --preset bundle-native +cpack --preset core # the core: runtime, C++ API, C binding +cpack --preset modules # the modules, without the core +``` + +The packages land in `build/bundle-native/packages`, named `opendaq---.tar.gz`. + +## Tests + +```bash +cmake -P tests/identity.cmake # the identity hash against its check vector +cmake -P tests/axis.cmake # axes and variants +cmake -S tests/fetchlayer-axes -B build/fetchlayer-axes # variants by two axes, the layer's decision +cmake -S tests/fetchlayer-git -B build/fetchlayer-git # the git origin (network) +cmake -S tests/fetchlayer -B build/fetchlayer -G Ninja \ + -DFETCHLAYER_STORE=$PWD/build/store -DFETCHLAYER_TRIPLET= # on the store, with a smoke test ``` diff --git a/bundle.json b/bundle.json new file mode 100644 index 0000000..42ff56e --- /dev/null +++ b/bundle.json @@ -0,0 +1,7 @@ +{ + "openDAQ": "1a82fa899b78e23ddcf61975eb9255b22e713808", + "OpcUaModules": "485710c37d06bcc6b6c852e57fd27c0b2cce7d08", + "MQTTStreamingModule": "3a55f4bf710ca10513ed64464e74680246e35e9f", + "LTStreamingModulesModern": "322d770b08e033eb6033d417c2310fdb832e088a", + "ASAM-CMP-Modules": "a95b8cea31cca2c5e437c7e55d38bc9efc7703a6" +} diff --git a/cmake/CPackOptions.cmake b/cmake/CPackOptions.cmake new file mode 100644 index 0000000..4df1213 --- /dev/null +++ b/cmake/CPackOptions.cmake @@ -0,0 +1,43 @@ +## +## Read by cpack for every generator (CPACK_PROJECT_CONFIG_FILE): an installer puts the bundle +## under /opt/opendaq, where the core installs itself; an archive holds the tree as it is. +## +## The package preset picks the stagings, BUNDLE_PACK_STAGINGS, and their components, BUNDLE_PACK_COMPONENTS +## (Runtime, Development): the install components Packing.cmake recorded for each, CPACK_BUNDLE__, +## the staging's name made a C identifier. The utils' config then names the package by CPACK_OPENDAQ_META_PACKAGE_NAME and writes its +## staging-meta.json. +## + +if (CPACK_GENERATOR MATCHES "^(DEB|RPM|productbuild)$") + set(CPACK_PACKAGING_INSTALL_PREFIX "/opt/opendaq") +else() + set(CPACK_PACKAGING_INSTALL_PREFIX "/") +endif() + +if (NOT BUNDLE_PACK_STAGINGS OR NOT BUNDLE_PACK_COMPONENTS) + message(FATAL_ERROR "cpack: pick what to pack, -D BUNDLE_PACK_STAGINGS=… -D BUNDLE_PACK_COMPONENTS=Runtime… " + "(a package preset does); the build has: ${CPACK_COMPONENTS_ALL}") +endif() +set(_picked) +foreach(_staging IN LISTS BUNDLE_PACK_STAGINGS) + string(MAKE_C_IDENTIFIER "${_staging}" _c) + set(_found FALSE) + foreach(_component IN LISTS BUNDLE_PACK_COMPONENTS) + if (CPACK_BUNDLE_${_c}_${_component}) + list(APPEND _picked ${CPACK_BUNDLE_${_c}_${_component}}) + set(_found TRUE) + endif() + endforeach() + if (NOT _found) + message(FATAL_ERROR "cpack: nothing of ${_staging} to pack (${BUNDLE_PACK_COMPONENTS}); the build has: ${CPACK_COMPONENTS_ALL}") + endif() +endforeach() +# a macOS package carries the bundle's uninstall script (Packing.cmake) +if (CPACK_GENERATOR STREQUAL "productbuild") + list(APPEND _picked bundle_Runtime) +endif() +set(CPACK_COMPONENTS_ALL ${_picked}) + +include("${CPACK_OPENDAQ_PROJECT_CONFIG}") +set(CPACK_PACKAGE_NAME "${CPACK_OPENDAQ_META_PACKAGE_NAME}") +set(CPACK_DEBIAN_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.deb") diff --git a/cmake/FetchLayer.cmake b/cmake/FetchLayer.cmake new file mode 100644 index 0000000..7ef20bf --- /dev/null +++ b/cmake/FetchLayer.cmake @@ -0,0 +1,330 @@ +## +## FetchLayer — FetchContent for layers, its sources swapped for prebuilt stagings where they exist. A layer is a +## repository at a revision; a staging is what a layer gives as one whole, with its own dependencies and identity. +## Stagings are made available, for one or more triplets, from where the policy says — prebuilt (System, later +## OCI) or built from their layer's sources (Git). +## +## FetchLayer_Declare( REVISION […] [POST_FETCH_INCLUDES