[Reference] cpp infra abstraction

.github/cloud_builder/run_command_on_active_checkout.yaml

@@ -3,7 +3,7 @@ substitutions:
 options:
   logging: CLOUD_LOGGING_ONLY
 steps:
-  - name: us-central1-docker.pkg.dev/external-snap-ci-github-gigl/gigl-base-images/gigl-builder:b34c863a2168c8df5a6da1f6385e5d374f0175d2.91.1
+  - name: us-central1-docker.pkg.dev/external-snap-ci-github-gigl/gigl-base-images/gigl-builder:3738af6cca02750514278cb63c98c88d07c45f7b.99.1
     entrypoint: /bin/bash
     args:
       - -c
@@ -28,8 +28,9 @@ steps:
         docker buildx create --driver=docker-container --use
         docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
         gcloud auth configure-docker us-central1-docker.pkg.dev
-        # Install GiGL
-        uv pip install -e .
+        # Install GiGL (non-editable). gigl-core (the C++ extension workspace member)
+        # is resolved transitively and built via scikit-build-core.
+        uv pip install .
         # The builder operates in its own user dir, usually /workspace,
         # so we need to copy the gigl tools dir to the current cloud_builder's user dir.
         # See: containers/Dockerfile.builder.

.github/workflows/on-pr-comment.yml

@@ -64,6 +64,23 @@ jobs:
         command: |
           make unit_test_py
 
+  unit-test-cpp:
+    if: ${{ github.event.issue.pull_request && (contains(github.event.comment.body, '/unit_test_cpp') || endsWith(github.event.comment.body, '/unit_test') || contains(github.event.comment.body, '/all_test')) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+    - name: Run C++ Unit Tests
+      uses: snapchat/gigl/.github/actions/run-command-on-pr@main
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        pr_number: ${{ github.event.issue.number }}
+        should_leave_progress_comments: "true"
+        descriptive_workflow_name: "C++ Unit Test"
+        use_cloud_run: "false"
+        command: |
+          bash requirements/install_cpp_deps.sh
+          make unit_test_cpp
+
   unit-test-scala:
     if: ${{ github.event.issue.pull_request && (contains(github.event.comment.body, '/unit_test_scala') || endsWith(github.event.comment.body, '/unit_test') || contains(github.event.comment.body, '/all_test')) }}
     runs-on: ubuntu-latest

.github/workflows/on-pr-merge.yml

@@ -70,6 +70,16 @@ jobs:
         service_account:  ${{ secrets.gcp_service_account_email }}
         project:  ${{ vars.GCP_PROJECT_ID }}
 
+  ci-unit-test-cpp:
+    if: github.event_name == 'merge_group'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install C++ dependencies
+      run: bash requirements/install_cpp_deps.sh
+    - name: Run C++ Unit Tests
+      run: make unit_test_cpp
+
   ci-integration-test:
     if: github.event_name == 'merge_group'
     runs-on: ubuntu-latest

.github/workflows/release-documentation.yml

@@ -41,11 +41,13 @@ jobs:
             gcp_project_id: ${{ vars.GCP_PROJECT_ID }}
             workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
             gcp_service_account_email: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
-      # We also make gigl available w/ editable install `-e` so that autodoc can find it.
+      # Install gigl so Sphinx autoapi can import and introspect it. Non-editable
+      # install; autodoc reads from site-packages. gigl-core (the C++ extension
+      # workspace member) is resolved transitively and built via scikit-build-core.
       - name: Install necessary doc dependencies
         run: |
           uv sync --group docs --inexact
-          uv pip install -e .
+          uv pip install .
       - name: Sphinx build
         run: |
           make build_docs

.github/workflows/release.yml

@@ -1,5 +1,10 @@
 name: Release GiGL
 
+# TODO: Before the first release using this workflow, complete the following one-time GCP setup:
+#   1. Create the `gigl-cu128` Artifact Registry Python repository in GCP project
+#      `external-snap-ci-github-gigl` (the CPU registry `gigl` already exists).
+#   2. Grant the release service account write access to the new `gigl-cu128` repository.
+
 on:
   # Triggers the workflow manually for now until we have full support for releasing:
   # - building and releasing docker images
@@ -11,13 +16,21 @@ permissions:
   id-token: write # Needed for the auth w/ GCP to upload to Google Artifact Registry.
 
 jobs:
-  build:
-    name: Build and release pip whl
-    runs-on: ubuntu-latest
+  build-gigl-core:
+    name: Build and release gigl-core (${{ matrix.torch-variant }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            torch-variant: cpu
+            index-name: gcp-release-registry-cpu
+          - runner: gigl-gpu-instances
+            torch-variant: cu128
+            index-name: gcp-release-registry-cu128
     env:
       PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
     environment:
-      # This CI environment contains relevant pip.conf and pyprci information to
       name: release
     steps:
     - name: Checkout
@@ -31,29 +44,55 @@ jobs:
           gcp_project_id: ${{ vars.GCP_PROJECT_ID }}
           workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
           gcp_service_account_email: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
-    # We need build and twine to build the whl and upload it to Google Artifact Registry.
-    # keyrings.google-artifactregistry-auth is needed to authenticate with Google Artifact Registry.
-    # See: https://cloud.google.com/artifact-registry/docs/python/store-python
-    # And: https://cloud.google.com/artifact-registry/docs/python/authentication
+
     - name: Setup environment for publishing Python package
       run: |
-        # Pre-install keyring and Artifact Registry plugin from the public PyPI
         uv tool install keyring --with keyrings.google-artifactregistry-auth==1.1.2
 
-    - name: Build Whl Distribution
-      run: uv build
+    - name: Build gigl-core Whl Distribution
+      run: uv build --wheel --package gigl-core
+
+    - name: Publish gigl-core Package 🚀
+      run: |
+        uv publish --index ${{ matrix.index-name }} --username oauth2accesstoken --keyring-provider subprocess
+
+  build:
+    name: Build and release gigl (${{ matrix.torch-variant }})
+    needs: build-gigl-core
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            torch-variant: cpu
+            index-name: gcp-release-registry-cpu
+          - runner: gigl-gpu-instances
+            torch-variant: cu128
+            index-name: gcp-release-registry-cu128
+    env:
+      PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
+    environment:
+      name: release
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Setup Python deps and gcloud
+      uses: ./.github/actions/setup-python-tools
+      with:
+          install_dev_deps: "true"
+          setup_gcloud: "true"
+          gcp_project_id: ${{ vars.GCP_PROJECT_ID }}
+          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+          gcp_service_account_email: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
 
-    - name: Publish Package 🚀
-      env:
-        PYPIRC_CONTENTS: ${{ secrets.PYPIRC_CONTENTS }}
-        PIP_CONF_CONTENTS: ${{ secrets.PIP_CONF_CONTENTS }}
-      # We upload the build whls to Google Artifact Registry.
+    - name: Setup environment for publishing Python package
       run: |
-        uv publish --index gcp-release-registry --username oauth2accesstoken --keyring-provider subprocess
+        uv tool install keyring --with keyrings.google-artifactregistry-auth==1.1.2
+
+    - name: Build gigl Whl Distribution
+      run: uv build --wheel --package gigl
 
-    - name: Post Publish Package
-      if: always()
-      # Clean up files created during Publish Package step.
+    - name: Publish gigl Package 🚀
       run: |
-        rm -rf ~/.pypirc
-        rm -rf ~/.pip/pip.conf
+        uv publish --index ${{ matrix.index-name }} --username oauth2accesstoken --keyring-provider subprocess

.gitignore

@@ -49,5 +49,22 @@ fossa*.zip
 # https://github.com/google-github-actions/auth/issues/497
 gha-creds-*.json
 
+# gigl-core build outputs. scikit-build-core writes the proper wheel build into
+# gigl-core/build/{wheel_tag}/; the editable-install cmake invocation additionally
+# drops cache files at gigl-core/ root (CMakeCache.txt, .cmake/, build.ninja, etc).
+gigl-core/build/
+gigl-core/CMakeCache.txt
+gigl-core/CMakeFiles/
+gigl-core/CMakeInit.txt
+gigl-core/cmake_install.cmake
+gigl-core/build.ninja
+gigl-core/.cmake/
+gigl-core/.ninja_log
+gigl-core/.ninja_deps
+gigl-core/.skbuild-info.json
+gigl-core/compile_commands.json
+gigl-core/_core*.so
+gigl-core/src/gigl_core/*.so
+
 # Local-only scripts with hardcoded internal identifiers
 scripts/_local/

Makefile

@@ -22,6 +22,13 @@ DOCKER_IMAGE_MAIN_CPU_NAME_WITH_TAG?=${DOCKER_IMAGE_MAIN_CPU_NAME}:${DATE}
 DOCKER_IMAGE_DEV_WORKBENCH_NAME_WITH_TAG?=${DOCKER_IMAGE_DEV_WORKBENCH_NAME}:${DATE}
 
 PYTHON_DIRS:=.github/scripts examples gigl tests snapchat scripts
+CPP_SOURCES:=$(shell find gigl-core/src gigl-core/tests \( -name "*.cpp" -o -name "*.cu" -o -name "*.h" -o -name "*.cuh" \) 2>/dev/null)
+# clang-tidy 15 does not fully support CUDA syntax (e.g. <<<...>>>, __global__).
+# Exclude .cu/.cuh files from tidy targets; clang-format and clangd handle them fine.
+CPP_SOURCES_NO_CUDA:=$(filter-out %.cu %.cuh,$(CPP_SOURCES))
+# scikit-build-core writes compile_commands.json into gigl-core/build/{wheel_tag}/
+# during `uv sync`. Resolved lazily at recipe time so the wildcard sees the latest build.
+CPP_COMPILE_COMMANDS_DIR=$(firstword $(wildcard gigl-core/build/*/))
 PY_TEST_FILES?="*_test.py"
 # You can override GIGL_TEST_DEFAULT_RESOURCE_CONFIG by setting it in your environment i.e.
 # adding `export GIGL_TEST_DEFAULT_RESOURCE_CONFIG=your_resource_config` to your shell config (~/.bashrc, ~/.zshrc, etc.)
@@ -48,7 +55,8 @@ check_if_valid_env:
 install_dev_deps: check_if_valid_env
 	gcloud auth configure-docker us-central1-docker.pkg.dev
 	bash ./requirements/install_py_deps.sh --dev
-	bash ./requirements/install_scala_deps.sh
+	# bash ./requirements/install_scala_deps.sh
+	# bash ./requirements/install_cpp_deps.sh
 	uv pip install -e .
 	uv run pre-commit install --hook-type pre-commit --hook-type pre-push
 
@@ -86,6 +94,15 @@ unit_test_scala: clean_build_files_scala
 	( cd scala; sbt test )
 	( cd scala_spark35 ; sbt test )
 
+# Builds and runs gigl-core's C++ unit tests (GoogleTest + CTest). scikit-build-core
+# is not involved here — we invoke cmake directly against gigl-core/ so we can
+# flip -DGIGL_CORE_BUILD_TESTS=ON to wire in the tests subdirectory.
+unit_test_cpp:
+	cd gigl-core && uv run cmake -S . -B build/tests -DGIGL_CORE_BUILD_TESTS=ON \
+		-DCMAKE_PREFIX_PATH=$$(uv run python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")
+	cd gigl-core && uv run cmake --build build/tests --parallel
+	cd gigl-core && uv run ctest --test-dir build/tests --output-on-failure
+
 # Runs unit tests for Python and Scala
 # Asserts Python and Scala files are formatted correctly.
 # Asserts YAML configs can be parsed.
@@ -94,7 +111,8 @@ unit_test_scala: clean_build_files_scala
 # Eventually, we should look into splitting these up.
 # We run `make check_format` separately instead of as a dependent make rule so that it always runs after the actual testing.
 # We don't want to fail the tests due to non-conformant formatting during development.
-unit_test: precondition_tests unit_test_py unit_test_scala
+
+unit_test: precondition_tests unit_test_py unit_test_scala unit_test_cpp
 
 check_format_py:
 	uv run ruff check --config pyproject.toml ${PYTHON_DIRS}
@@ -108,7 +126,12 @@ check_format_md:
 	@echo "Checking markdown files..."
 	uv run mdformat --check ${MD_FILES}
 
-check_format: check_format_py check_format_scala check_format_md
+check_format_cpp:
+	clang-format-15 --dry-run --Werror --style=file $(CPP_SOURCES)
+
+# Checks formatting only (clang-format, black, scalafmt, mdformat). Does NOT run
+# clang-tidy static analysis — use `make check_lint_cpp` for that.
+check_format: check_format_py check_format_cpp check_format_scala check_format_md
 
 # Set PY_TEST_FILES=<TEST_FILE_NAME_GLOB> to test a specifc file.
 # Ex. `make integration_test PY_TEST_FILES="dataflow_test.py"`
@@ -141,14 +164,42 @@ format_md:
 	@echo "Formatting markdown files..."
 	uv run mdformat ${MD_FILES}
 
-format: format_py format_scala format_md
+format_cpp:
+	clang-format-15 -i --style=file $(CPP_SOURCES)
+
+format: format_py format_cpp format_scala format_md
 
 type_check:
 	uv run mypy ${PYTHON_DIRS} --check-untyped-defs
 
-lint_test: check_format assert_yaml_configs_parse
+marco:
+	echo "polo"
+	touch marco
+
+# scikit-build-core writes compile_commands.json into gigl-core/build/{wheel_tag}/
+# during `uv sync` / `uv pip install -e ./gigl-core`. clang-tidy reads it directly.
+# If the path is empty (gigl-core has never been built), `uv sync` populates it.
+check_lint_cpp:
+	@test -n "$(CPP_COMPILE_COMMANDS_DIR)" || (echo "No gigl-core build dir found. Run 'uv sync' first."; exit 1)
+	clang-tidy-15 -p $(CPP_COMPILE_COMMANDS_DIR) $(CPP_SOURCES_NO_CUDA)
+
+# Not part of `make format`: clang-tidy --fix rewrites logic (renames identifiers,
+# changes expressions, adds/removes keywords), not just style. Run manually and
+# review the diff before committing. Note: --fix cannot auto-repair every check;
+# some violations require manual edits.
+fix_lint_cpp:
+	@test -n "$(CPP_COMPILE_COMMANDS_DIR)" || (echo "No gigl-core build dir found. Run 'uv sync' first."; exit 1)
+	clang-tidy-15 --fix -p $(CPP_COMPILE_COMMANDS_DIR) $(CPP_SOURCES_NO_CUDA)
+
+lint_test: check_format assert_yaml_configs_parse check_lint_cpp
 	@echo "Lint checks pass!"
 
+# Wipe gigl-core's build cache. Use this if cmake's cached state becomes inconsistent
+# after switching between branches with substantially different CMakeLists.txt structure.
+clean_cpp:
+	rm -rf gigl-core/build
+	rm -f gigl-core/src/gigl_core/*.so
+
 # compiles current working state of scala projects to local jars
 compile_jars:
 	@echo "Compiling jars..."
@@ -311,7 +362,10 @@ clean_build_files_scala:
 	( cd scala; sbt clean; find . -type d -name "target" -prune -exec rm -rf {} \; )
 	( cd scala_spark35; sbt clean; find . -type d -name "target" -prune -exec rm -rf {} \; )
 
-clean_build_files: clean_build_files_py clean_build_files_scala
+clean_build_files_cpp:
+	rm -rf gigl-core/build
+
+clean_build_files: clean_build_files_py clean_build_files_scala clean_build_files_cpp
 
 # Call to generate new proto definitions if any of the .proto files have been changed.
 # We intentionally rebuild *all* protos with one commmand as they should all be in sync.

containers/Dockerfile.dataflow.src

@@ -5,14 +5,17 @@ FROM $BASE_IMAGE
 # Copy the source
 WORKDIR /gigl
 
-COPY MANIFEST.in MANIFEST.in
-COPY pyproject.toml pyproject.toml
-COPY uv.lock uv.lock
+# Copy everything in one layer. The C++ extension lives in the separate `gigl-core`
+# workspace member; `uv pip install .` resolves it from the workspace and builds it
+# via scikit-build-core — no separate `make build_cpp_extensions` step needed.
+COPY pyproject.toml uv.lock Makefile README.md ./
+COPY gigl-core gigl-core
 COPY gigl/dep_vars.env gigl/dep_vars.env
 COPY deployment deployment
 COPY gigl gigl
 COPY snapchat snapchat
 COPY tests tests
-RUN uv pip install -e .
+
+RUN uv pip install .
 
 WORKDIR /

containers/Dockerfile.src

@@ -1,22 +1,20 @@
 ARG BASE_IMAGE
 FROM $BASE_IMAGE
 
-# Copy the source
-WORKDIR /gigl
-
-
 # Note: main package files must live in root of the repo for the python package to be built correctly for Dataflow workers.
 # See https://beam.apache.org/documentation/sdks/python-pipxeline-dependencies/#create-reproducible-environments.
 WORKDIR /gigl
 
-COPY MANIFEST.in MANIFEST.in
-COPY pyproject.toml pyproject.toml
-COPY uv.lock uv.lock
+# Copy everything in one layer. The C++ extension lives in the separate `gigl-core`
+# workspace member; `uv pip install .` resolves it from the workspace and builds it
+# via scikit-build-core — no separate `make build_cpp_extensions` step needed.
+COPY pyproject.toml uv.lock Makefile README.md ./
+COPY gigl-core gigl-core
 COPY gigl/dep_vars.env gigl/dep_vars.env
 COPY deployment deployment
 COPY gigl gigl
 COPY snapchat snapchat
 COPY tests tests
 COPY examples examples
 
-RUN uv pip install -e .
+RUN uv pip install .