From 6557f0916b7bbd40caa4990668af2329d771cee1 Mon Sep 17 00:00:00 2001
From: Marc Julien <marc@siftstack.com>
Date: Mon, 11 Nov 2024 13:26:56 -0800
Subject: [PATCH 01/75] Add job to build wheels

---
 .github/workflows/python_build.yaml | 85 +++++++++++++++++++++++++++++
 python/pyproject.toml               |  2 +-
 2 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/python_build.yaml

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
new file mode 100644
index 000000000..dd3f09f76
--- /dev/null
+++ b/.github/workflows/python_build.yaml
@@ -0,0 +1,85 @@
+name: Build Wheels
+
+on:
+  workflow_dispatch:
+
+jobs:
+  python-ci:
+    if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags')
+    uses: ./.github/workflows/python_ci.yaml
+
+  build_matrix:
+    strategy:
+      matrix:
+        python_version: [3.9, 3.11]
+
+    name: Build Python ${{ matrix.python_version }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: python
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Get build metadata
+        id: metadata
+        run: |
+          export VERSION="$(basename ${{ github.ref_name }})"
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+
+      - name: Create builds directory
+        working-directory: python
+        run: |
+          mkdir builds
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "${{ matrix.python_version }}"
+
+      - name: Pip install
+        run: |
+          python -m pip install --upgrade pip
+          pip install pip-tools
+
+      - name: Compile requirements
+        working-directory: python
+        run: |
+          pip-compile pyproject.toml --output-file requirements.txt
+
+      - name: Build Windows
+        working-directory: python
+        env:
+          PLATFORM: win_amd64
+          ARTIFACT_NAME: sift_stack_py-${{ steps.metadata.outputs.version }}-py${{ matrix.python_version }}-win_amd64
+        run: |
+          pip download -r requirements.txt -d dist --platform=${PLATFORM} --only-binary=:all: --python-version=${{ matrix.python_version }}
+          python setup.py bdist_wheel
+          zip ${ARTIFACT_NAME}.zip dist -r
+          sha1sum ${ARTIFACT_NAME}.zip | cut -f1 -d' ' > ${ARTIFACT_NAME}.sha1
+          mv ${ARTIFACT_NAME}.zip builds
+          mv ${ARTIFACT_NAME}.sha1 builds
+          rm -rf dist
+
+      - name: Build Linux
+        working-directory: python
+        env:
+          PLATFORM: manylinux_2_17_x86_64
+          ARTIFACT_NAME: sift_stack_py-${{ steps.metadata.outputs.version }}-py${{ matrix.python_version }}-manylinux_2_17_x86_64
+        run: |
+          pip download -r requirements.txt -d dist --platform=${PLATFORM} --only-binary=:all: --python-version=${{ matrix.python_version }}
+          python setup.py bdist_wheel
+          zip ${ARTIFACT_NAME}.zip dist -r
+          sha1sum ${ARTIFACT_NAME}.zip | cut -f1 -d' ' > ${ARTIFACT_NAME}.sha1
+          mv ${ARTIFACT_NAME}.zip builds
+          mv ${ARTIFACT_NAME}.sha1 builds
+          rm -rf dist
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: sift_stack_py-${{ steps.metadata.outputs.version }}-py${{ matrix.python_version }}
+          path: python/builds
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 1010474ab..94e49e884 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -22,7 +22,6 @@ keywords = [
 ]
 dependencies = [
   "grpcio~=1.13",
-  "npTDMS~=1.9",
   "PyYAML~=6.0",
   "pandas~=2.0",
   "protobuf>=4.0",
@@ -48,6 +47,7 @@ Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md"
 [project.optional-dependencies]
 development = [
   "grpcio-testing==1.13",
+  "npTDMS~=1.9",
   "mypy==1.10.0",
   "pyright==1.1.386",
   "pytest==8.2.2",

From 72b9acc672edfca1dde0a7dd5f923b9f8db1f154 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:21:27 -0800
Subject: [PATCH 02/75] Use cibuildwheel

also add .venv to gitignore
---
 .github/workflows/python_build.yaml | 153 ++++++++++++++++------------
 .gitignore                          |   1 +
 2 files changed, 91 insertions(+), 63 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index dd3f09f76..d74333b62 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -4,82 +4,109 @@ on:
   workflow_dispatch:
 
 jobs:
-  python-ci:
-    if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags')
-    uses: ./.github/workflows/python_ci.yaml
+  get-python-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      python-versions: ${{ steps.get-versions.outputs.versions }}
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Get Python versions from pyproject.toml
+        id: get-versions
+        run: |
+          versions=$(grep "Programming Language :: Python :: " python/pyproject.toml | sed 's/.*Python :: \([0-9.]*\).*/\1/' | jq -R -s -c 'split("\n")[:-1]')
+          echo "versions=$versions" >> $GITHUB_OUTPUT
 
-  build_matrix:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    needs: get-python-versions
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python_version: [3.9, 3.11]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
 
-    name: Build Python ${{ matrix.python_version }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: python
-    permissions:
-      id-token: write
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: Get build metadata
-        id: metadata
-        run: |
-          export VERSION="$(basename ${{ github.ref_name }})"
-          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
-      - name: Create builds directory
-        working-directory: python
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel
+
+      - name: Build wheels
+        env:
+          CIBW_BUILD: "cp*"
+          CIBW_SKIP: "cp36-* cp37-* *-win32 *-manylinux_i686 *-musllinux*"
+          CIBW_ARCHS_MACOS: "x86_64 arm64"
+          CIBW_ARCHS_LINUX: "x86_64"
+          CIBW_ARCHS_WINDOWS: "AMD64"
+          # Add any build dependencies your package needs
+          CIBW_BEFORE_BUILD: "pip install build"
+          # Test dependencies and commands if needed
+          CIBW_TEST_REQUIRES: "pytest"
+          CIBW_TEST_COMMAND: "pytest {project}/tests"
         run: |
-          mkdir builds
+          python -m cibuildwheel python --output-dir wheelhouse
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: ./wheelhouse/*.whl
+          retention-days: 7
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: "${{ matrix.python_version }}"
+          python-version: "3.8"
 
-      - name: Pip install
-        run: |
-          python -m pip install --upgrade pip
-          pip install pip-tools
+      - name: Install build
+        run: python -m pip install build
 
-      - name: Compile requirements
-        working-directory: python
-        run: |
-          pip-compile pyproject.toml --output-file requirements.txt
+      - name: Build sdist
+        run: python -m build --sdist python
 
-      - name: Build Windows
-        working-directory: python
-        env:
-          PLATFORM: win_amd64
-          ARTIFACT_NAME: sift_stack_py-${{ steps.metadata.outputs.version }}-py${{ matrix.python_version }}-win_amd64
-        run: |
-          pip download -r requirements.txt -d dist --platform=${PLATFORM} --only-binary=:all: --python-version=${{ matrix.python_version }}
-          python setup.py bdist_wheel
-          zip ${ARTIFACT_NAME}.zip dist -r
-          sha1sum ${ARTIFACT_NAME}.zip | cut -f1 -d' ' > ${ARTIFACT_NAME}.sha1
-          mv ${ARTIFACT_NAME}.zip builds
-          mv ${ARTIFACT_NAME}.sha1 builds
-          rm -rf dist
-
-      - name: Build Linux
-        working-directory: python
-        env:
-          PLATFORM: manylinux_2_17_x86_64
-          ARTIFACT_NAME: sift_stack_py-${{ steps.metadata.outputs.version }}-py${{ matrix.python_version }}-manylinux_2_17_x86_64
-        run: |
-          pip download -r requirements.txt -d dist --platform=${PLATFORM} --only-binary=:all: --python-version=${{ matrix.python_version }}
-          python setup.py bdist_wheel
-          zip ${ARTIFACT_NAME}.zip dist -r
-          sha1sum ${ARTIFACT_NAME}.zip | cut -f1 -d' ' > ${ARTIFACT_NAME}.sha1
-          mv ${ARTIFACT_NAME}.zip builds
-          mv ${ARTIFACT_NAME}.sha1 builds
-          rm -rf dist
-
-      - name: Upload artifact
+      - name: Upload sdist
         uses: actions/upload-artifact@v4
         with:
-          name: sift_stack_py-${{ steps.metadata.outputs.version }}-py${{ matrix.python_version }}
-          path: python/builds
+          name: sdist
+          path: python/dist/*.tar.gz
+          retention-days: 7
+
+  verify_wheels:
+    name: Verify wheels
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+
+      - name: Install twine
+        run: python -m pip install twine
+
+      - name: Check wheels
+        run: |
+          twine check dist/*
diff --git a/.gitignore b/.gitignore
index 9f7ad8e80..e7d1e7bef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 **/.env
 **/venv
+**/.venv
 **/__pycache__
 **/dist
 

From 5fe0be60ca0fd63f1f20b90b5593157cdaf8f4c4 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:24:11 -0800
Subject: [PATCH 03/75] trigger on pr for testing

---
 .github/workflows/python_build.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d74333b62..e49ea74c8 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -2,6 +2,7 @@ name: Build Wheels
 
 on:
   workflow_dispatch:
+  pull_request:
 
 jobs:
   get-python-versions:

From a3628b4dbfc80ec28e56c86ba9105b6a41d1067c Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:34:01 -0800
Subject: [PATCH 04/75] Enhance CI workflow for Python package builds and
 testing

- Updated the build dependencies to include optional extras for building.
- Modified the test command to validate both standard and optional dependencies.
- Expanded the wheel checking step to include installation tests for various combinations of extras.

This improves the robustness of the build process and ensures all features are tested.
---
 .github/workflows/python_build.yaml | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index e49ea74c8..1afc95a51 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -45,11 +45,20 @@ jobs:
           CIBW_ARCHS_MACOS: "x86_64 arm64"
           CIBW_ARCHS_LINUX: "x86_64"
           CIBW_ARCHS_WINDOWS: "AMD64"
-          # Add any build dependencies your package needs
-          CIBW_BEFORE_BUILD: "pip install build"
-          # Test dependencies and commands if needed
+          # Install all optional dependencies for building
+          CIBW_BEFORE_BUILD: >-
+            pip install build &&
+            pip install .[openssl,build,dev]
+          # Test dependencies and commands
           CIBW_TEST_REQUIRES: "pytest"
-          CIBW_TEST_COMMAND: "pytest {project}/tests"
+          # Test both with and without optional dependencies
+          CIBW_TEST_COMMAND: >-
+            pip install {package} &&
+            pytest {project}/tests &&
+            pip install {package}[openssl] &&
+            pytest {project}/tests
+          # Ensure all extras are included in the wheel
+          CIBW_BUILD_FRONTEND: "pip; args: --config-settings=--build-option=--with-openssl"
         run: |
           python -m cibuildwheel python --output-dir wheelhouse
 
@@ -108,6 +117,12 @@ jobs:
       - name: Install twine
         run: python -m pip install twine
 
-      - name: Check wheels
+      - name: Check wheels and test installation
         run: |
           twine check dist/*
+          # Test installation with different combinations of extras
+          python -m pip install --force-reinstall dist/*.whl
+          python -m pip install --force-reinstall dist/*.whl[openssl]
+          python -m pip install --force-reinstall dist/*.whl[build]
+          python -m pip install --force-reinstall dist/*.whl[dev]
+          python -m pip install --force-reinstall dist/*.whl[openssl,build,dev]

From 3697bb91897a4cbfbe49e69c6b18da755f516912 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:36:48 -0800
Subject: [PATCH 05/75] fixup

---
 .github/workflows/python_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 1afc95a51..d6fd2b421 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -26,6 +26,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
+        fail-fast: false
 
     steps:
       - uses: actions/checkout@v4
@@ -48,7 +49,7 @@ jobs:
           # Install all optional dependencies for building
           CIBW_BEFORE_BUILD: >-
             pip install build &&
-            pip install .[openssl,build,dev]
+            pip install -e {project}[openssl,build,dev]
           # Test dependencies and commands
           CIBW_TEST_REQUIRES: "pytest"
           # Test both with and without optional dependencies

From cd5cbe56fb9ededdc887315b04bd228339e7bf14 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:38:02 -0800
Subject: [PATCH 06/75] fixup

---
 .github/workflows/python_build.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d6fd2b421..2c905c566 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -26,7 +26,6 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
-        fail-fast: false
 
     steps:
       - uses: actions/checkout@v4

From a4d8f6f0f7f6d3402cf66e26ee6baf65b4e2a004 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:40:29 -0800
Subject: [PATCH 07/75] set working directory

---
 .github/workflows/python_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 2c905c566..573ba67dd 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -48,7 +48,7 @@ jobs:
           # Install all optional dependencies for building
           CIBW_BEFORE_BUILD: >-
             pip install build &&
-            pip install -e {project}[openssl,build,dev]
+            pip install -e .[openssl,build,dev]
           # Test dependencies and commands
           CIBW_TEST_REQUIRES: "pytest"
           # Test both with and without optional dependencies
@@ -61,6 +61,7 @@ jobs:
           CIBW_BUILD_FRONTEND: "pip; args: --config-settings=--build-option=--with-openssl"
         run: |
           python -m cibuildwheel python --output-dir wheelhouse
+        working-directory: python
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4

From b999b6ac7022c800d9f31c0100f7b9cb4b0191ef Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:43:47 -0800
Subject: [PATCH 08/75] some debugging

---
 .github/workflows/python_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 573ba67dd..01a793777 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -36,7 +36,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel
+        run: python -m pip install cibuildwheel && ls -la
 
       - name: Build wheels
         env:
@@ -47,6 +47,7 @@ jobs:
           CIBW_ARCHS_WINDOWS: "AMD64"
           # Install all optional dependencies for building
           CIBW_BEFORE_BUILD: >-
+            cd python &&
             pip install build &&
             pip install -e .[openssl,build,dev]
           # Test dependencies and commands

From 4c3115f347d90aab97826525e6b5e1bc0b387308 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:45:21 -0800
Subject: [PATCH 09/75] more trial and error

---
 .github/workflows/python_build.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 01a793777..7ae90ba80 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -62,7 +62,6 @@ jobs:
           CIBW_BUILD_FRONTEND: "pip; args: --config-settings=--build-option=--with-openssl"
         run: |
           python -m cibuildwheel python --output-dir wheelhouse
-        working-directory: python
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4

From 01efca0b8faae98174d783fde7d448dc08ba2aa0 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:46:57 -0800
Subject: [PATCH 10/75] ugh

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 7ae90ba80..80b9f3d22 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -36,7 +36,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel && ls -la
+        run: python -m pip install cibuildwheel
 
       - name: Build wheels
         env:

From 21d54a12035737d24b2fbbd64f63eb4ab46c4865 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 15:50:34 -0800
Subject: [PATCH 11/75] fix up more

---
 .github/workflows/python_build.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 80b9f3d22..606f43658 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -58,8 +58,6 @@ jobs:
             pytest {project}/tests &&
             pip install {package}[openssl] &&
             pytest {project}/tests
-          # Ensure all extras are included in the wheel
-          CIBW_BUILD_FRONTEND: "pip; args: --config-settings=--build-option=--with-openssl"
         run: |
           python -m cibuildwheel python --output-dir wheelhouse
 

From 580892153d782e7df309fae88951e181b0693248 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 23:05:16 -0800
Subject: [PATCH 12/75] Refactor CI workflow for Python package builds and
 testing

- Renamed workflow from "Build Wheels" to "Offline Installation Archive" to better reflect its purpose.
- Updated job names and structure to enhance clarity and organization.
- Added support for multiple architectures and operating systems in the build matrix.
- Introduced steps for generating requirements, downloading dependencies, and creating a distribution archive.
- Enhanced verification process by extracting and checking distributions before testing installations.
- Improved installation testing to cover various combinations of optional dependencies.

These changes streamline the CI process and ensure comprehensive testing of the package across different environments.
---
 .github/workflows/python_build.yaml | 155 +++++++++++++++-------------
 1 file changed, 86 insertions(+), 69 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 606f43658..b0e25e68b 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -1,4 +1,4 @@
-name: Build Wheels
+name: Offline Installation Archive
 
 on:
   workflow_dispatch:
@@ -18,96 +18,98 @@ jobs:
           versions=$(grep "Programming Language :: Python :: " python/pyproject.toml | sed 's/.*Python :: \([0-9.]*\).*/\1/' | jq -R -s -c 'split("\n")[:-1]')
           echo "versions=$versions" >> $GITHUB_OUTPUT
 
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
+  build_packages:
+    name: Build on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-python-versions
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        include:
+          # Ubuntu builds (x86_64 and aarch64)
+          - os: ubuntu
+            arch: x86_64
+            runner: ubuntu-latest
+          - os: ubuntu
+            arch: aarch64
+            runner: ubuntu-latest
+            
+          # macOS builds (x86_64 and arm64)
+          - os: macos
+            arch: x86_64
+            runner: macos-latest
+          - os: macos
+            arch: arm64
+            runner: macos-14
+            
+          # Windows builds (x86 and x64)
+          - os: windows
+            arch: x86
+            runner: windows-latest
+          - os: windows
+            arch: x64
+            runner: windows-latest
         python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
 
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        if: matrix.arch == 'aarch64'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel
-
-      - name: Build wheels
-        env:
-          CIBW_BUILD: "cp*"
-          CIBW_SKIP: "cp36-* cp37-* *-win32 *-manylinux_i686 *-musllinux*"
-          CIBW_ARCHS_MACOS: "x86_64 arm64"
-          CIBW_ARCHS_LINUX: "x86_64"
-          CIBW_ARCHS_WINDOWS: "AMD64"
-          # Install all optional dependencies for building
-          CIBW_BEFORE_BUILD: >-
-            cd python &&
-            pip install build &&
-            pip install -e .[openssl,build,dev]
-          # Test dependencies and commands
-          CIBW_TEST_REQUIRES: "pytest"
-          # Test both with and without optional dependencies
-          CIBW_TEST_COMMAND: >-
-            pip install {package} &&
-            pytest {project}/tests &&
-            pip install {package}[openssl] &&
-            pytest {project}/tests
+      - name: Install build tools
         run: |
-          python -m cibuildwheel python --output-dir wheelhouse
+          python -m pip install --upgrade pip
+          pip install build pip-tools
 
-      - name: Upload wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.os }}-py${{ matrix.python-version }}
-          path: ./wheelhouse/*.whl
-          retention-days: 7
-
-  build_sdist:
-    name: Build source distribution
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
+      - name: Generate requirements
+        working-directory: python
+        run: |
+          pip-compile pyproject.toml --extra=openssl,build,dev -o requirements-all.txt
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.8"
+      - name: Download dependencies
+        working-directory: python
+        env:
+          PIP_PLATFORM: ${{ matrix.os == 'windows' && format('{0}_win32', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
+        run: |
+          mkdir -p dist/deps
+          pip download -r requirements-all.txt -d dist/deps --platform $PIP_PLATFORM --only-binary=:all:
 
-      - name: Install build
-        run: python -m pip install build
+      - name: Build sift-py wheel
+        working-directory: python
+        run: |
+          python -m build --wheel
 
-      - name: Build sdist
-        run: python -m build --sdist python
+      - name: Create distribution archive
+        working-directory: python
+        run: |
+          python -c "import shutil; shutil.make_archive('dist/sift-py-dist-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python-version }}', 'zip', 'dist')"
 
-      - name: Upload sdist
+      - name: Upload distribution archive
         uses: actions/upload-artifact@v4
         with:
-          name: sdist
-          path: python/dist/*.tar.gz
+          name: sift-py-dist-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python-version }}
+          path: python/dist/sift-py-dist-*.zip
           retention-days: 7
 
-  verify_wheels:
-    name: Verify wheels
-    needs: [build_wheels, build_sdist]
+  verify_distributions:
+    name: Verify distributions
+    needs: build_packages
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v4
         with:
-          pattern: wheels-*
+          pattern: sift-py-dist-*
           path: dist
           merge-multiple: true
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: sdist
-          path: dist
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -116,12 +118,27 @@ jobs:
       - name: Install twine
         run: python -m pip install twine
 
-      - name: Check wheels and test installation
+      - name: Extract and verify distributions
+        run: |
+          # Extract all zip files
+          for zip in dist/*.zip; do
+            unzip -o "$zip" -d "./extracted/$(basename "$zip" .zip)"
+          done
+          
+          # Check all wheels with twine
+          find ./extracted -name "*.whl" -exec twine check {} +
+
+      - name: Test installations
         run: |
-          twine check dist/*
-          # Test installation with different combinations of extras
-          python -m pip install --force-reinstall dist/*.whl
-          python -m pip install --force-reinstall dist/*.whl[openssl]
-          python -m pip install --force-reinstall dist/*.whl[build]
-          python -m pip install --force-reinstall dist/*.whl[dev]
-          python -m pip install --force-reinstall dist/*.whl[openssl,build,dev]
+          # Test installation from each distribution
+          for dist_dir in ./extracted/*; do
+            echo "Testing installation from $dist_dir"
+            # Install the main package wheel with different combinations of extras
+            main_wheel=$(find "$dist_dir" -name "sift_stack_py*.whl")
+            pip install --no-index --find-links="$dist_dir" "$main_wheel"
+            pip install --no-index --find-links="$dist_dir" "$main_wheel[openssl]"
+            pip install --no-index --find-links="$dist_dir" "$main_wheel[build]"
+            pip install --no-index --find-links="$dist_dir" "$main_wheel[dev]"
+            pip install --no-index --find-links="$dist_dir" "$main_wheel[openssl,build,dev]"
+            pip uninstall -y sift_stack_py
+          done

From 5bb0910723cffba191120abb16562fd3be13f4fc Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 23:28:16 -0800
Subject: [PATCH 13/75] Update grpcio-testing dependency version in
 pyproject.toml to use compatible versioning syntax

---
 python/pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyproject.toml b/python/pyproject.toml
index 94e49e884..e792e77cc 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -46,7 +46,7 @@ Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md"
 
 [project.optional-dependencies]
 development = [
-  "grpcio-testing==1.13",
+  "grpcio-testing~=1.13",
   "npTDMS~=1.9",
   "mypy==1.10.0",
   "pyright==1.1.386",

From 0c367b45fe86a09be5c82c807d09dce5779714ab Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 23:29:18 -0800
Subject: [PATCH 14/75] update the download command

---
 .github/workflows/python_build.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index b0e25e68b..7bf3e174d 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -72,7 +72,7 @@ jobs:
       - name: Generate requirements
         working-directory: python
         run: |
-          pip-compile pyproject.toml --extra=openssl,build,dev -o requirements-all.txt
+          pip-compile pyproject.toml --all-extras -o requirements-all.txt
 
       - name: Download dependencies
         working-directory: python
@@ -80,7 +80,11 @@ jobs:
           PIP_PLATFORM: ${{ matrix.os == 'windows' && format('{0}_win32', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
         run: |
           mkdir -p dist/deps
-          pip download -r requirements-all.txt -d dist/deps --platform $PIP_PLATFORM --only-binary=:all:
+          # Download all wheels
+          pip download -r requirements-all.txt -d dist/deps \
+            --platform $PIP_PLATFORM \
+            --python-version ${{ matrix.python-version }} \
+            --no-deps
 
       - name: Build sift-py wheel
         working-directory: python

From a8db69cc26a4c54bbadf4ef60e8471861125f497 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 23:42:09 -0800
Subject: [PATCH 15/75] Refactor CI workflow to enhance build and verification
 process for Python packages. Renamed job to "Build and verify", added
 verification step for wheel files using twine, and implemented comprehensive
 installation tests for various dependency combinations. Updated installation
 steps and removed obsolete verification job to streamline the workflow.

---
 .github/workflows/python_build.yaml | 88 +++++++++++++----------------
 1 file changed, 40 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 7bf3e174d..cccd2513c 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -18,8 +18,8 @@ jobs:
           versions=$(grep "Programming Language :: Python :: " python/pyproject.toml | sed 's/.*Python :: \([0-9.]*\).*/\1/' | jq -R -s -c 'split("\n")[:-1]')
           echo "versions=$versions" >> $GITHUB_OUTPUT
 
-  build_packages:
-    name: Build on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
+  build_and_verify:
+    name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-python-versions
     runs-on: ${{ matrix.runner }}
     strategy:
@@ -65,22 +65,24 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install build tools
+        shell: bash
         run: |
           python -m pip install --upgrade pip
-          pip install build pip-tools
+          pip install build pip-tools twine
 
       - name: Generate requirements
         working-directory: python
+        shell: bash
         run: |
           pip-compile pyproject.toml --all-extras -o requirements-all.txt
 
       - name: Download dependencies
         working-directory: python
+        shell: bash
         env:
           PIP_PLATFORM: ${{ matrix.os == 'windows' && format('{0}_win32', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
         run: |
           mkdir -p dist/deps
-          # Download all wheels
           pip download -r requirements-all.txt -d dist/deps \
             --platform $PIP_PLATFORM \
             --python-version ${{ matrix.python-version }} \
@@ -88,11 +90,45 @@ jobs:
 
       - name: Build sift-py wheel
         working-directory: python
+        shell: bash
         run: |
           python -m build --wheel
 
+      - name: Verify wheel
+        working-directory: python
+        shell: bash
+        run: |
+          twine check dist/*.whl
+
+      - name: Test installations
+        working-directory: python
+        shell: bash
+        run: |
+          # Function to create venv and test installation
+          test_install() {
+            local extras=$1
+            echo "Testing installation with extras: ${extras:-none}"
+            python -m venv test_venv
+            source test_venv/bin/activate || source test_venv/Scripts/activate
+            if [ -z "$extras" ]; then
+              pip install --no-index --find-links=dist/deps dist/*.whl
+            else
+              pip install --no-index --find-links=dist/deps "dist/*.whl[$extras]"
+            fi
+            deactivate
+            rm -rf test_venv
+          }
+
+          # Test each combination in a fresh venv
+          test_install ""  # no extras
+          test_install "openssl"
+          test_install "build"
+          test_install "dev"
+          test_install "openssl,build,dev"
+
       - name: Create distribution archive
         working-directory: python
+        shell: bash
         run: |
           python -c "import shutil; shutil.make_archive('dist/sift-py-dist-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python-version }}', 'zip', 'dist')"
 
@@ -102,47 +138,3 @@ jobs:
           name: sift-py-dist-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python-version }}
           path: python/dist/sift-py-dist-*.zip
           retention-days: 7
-
-  verify_distributions:
-    name: Verify distributions
-    needs: build_packages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: sift-py-dist-*
-          path: dist
-          merge-multiple: true
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.8"
-
-      - name: Install twine
-        run: python -m pip install twine
-
-      - name: Extract and verify distributions
-        run: |
-          # Extract all zip files
-          for zip in dist/*.zip; do
-            unzip -o "$zip" -d "./extracted/$(basename "$zip" .zip)"
-          done
-          
-          # Check all wheels with twine
-          find ./extracted -name "*.whl" -exec twine check {} +
-
-      - name: Test installations
-        run: |
-          # Test installation from each distribution
-          for dist_dir in ./extracted/*; do
-            echo "Testing installation from $dist_dir"
-            # Install the main package wheel with different combinations of extras
-            main_wheel=$(find "$dist_dir" -name "sift_stack_py*.whl")
-            pip install --no-index --find-links="$dist_dir" "$main_wheel"
-            pip install --no-index --find-links="$dist_dir" "$main_wheel[openssl]"
-            pip install --no-index --find-links="$dist_dir" "$main_wheel[build]"
-            pip install --no-index --find-links="$dist_dir" "$main_wheel[dev]"
-            pip install --no-index --find-links="$dist_dir" "$main_wheel[openssl,build,dev]"
-            pip uninstall -y sift_stack_py
-          done

From c97b46a2536d2733f2cc2567aa2a1a49da90d6e6 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Mon, 9 Dec 2024 23:48:13 -0800
Subject: [PATCH 16/75] use --prefer-binary

---
 .github/workflows/python_build.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index cccd2513c..12cd585fa 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -83,9 +83,11 @@ jobs:
           PIP_PLATFORM: ${{ matrix.os == 'windows' && format('{0}_win32', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
         run: |
           mkdir -p dist/deps
+          # Try to download all packages, preferring wheels but allowing source if wheels aren't available
           pip download -r requirements-all.txt -d dist/deps \
             --platform $PIP_PLATFORM \
             --python-version ${{ matrix.python-version }} \
+            --prefer-binary \
             --no-deps
 
       - name: Build sift-py wheel

From af8af2daa13906fe19a9452760f6659bfa87d036 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:05:24 -0800
Subject: [PATCH 17/75] get rid of 32bit windows builds

---
 .github/workflows/python_build.yaml | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 12cd585fa..3fdcd1b6b 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -22,6 +22,8 @@ jobs:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-python-versions
     runs-on: ${{ matrix.runner }}
+    env:
+      PIP_PLATFORM: ${{ matrix.os == 'windows' && format('win_{0}', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
     strategy:
       matrix:
         include:
@@ -41,12 +43,9 @@ jobs:
             arch: arm64
             runner: macos-14
             
-          # Windows builds (x86 and x64)
+          # Windows builds (x64)
           - os: windows
-            arch: x86
-            runner: windows-latest
-          - os: windows
-            arch: x64
+            arch: amd64
             runner: windows-latest
         python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
 
@@ -55,7 +54,7 @@ jobs:
 
       - name: Set up QEMU
         if: matrix.arch == 'aarch64'
-        uses: docker/setup-qemu-action@v3
+       uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
 
@@ -79,8 +78,6 @@ jobs:
       - name: Download dependencies
         working-directory: python
         shell: bash
-        env:
-          PIP_PLATFORM: ${{ matrix.os == 'windows' && format('{0}_win32', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
         run: |
           mkdir -p dist/deps
           # Try to download all packages, preferring wheels but allowing source if wheels aren't available
@@ -132,11 +129,11 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          python -c "import shutil; shutil.make_archive('dist/sift-py-dist-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python-version }}', 'zip', 'dist')"
+          python -c "import shutil; shutil.make_archive('dist/sift-py-dist-${{ env.PIP_PLATFORM }}-py${{ matrix.python-version }}', 'zip', 'dist')"
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python-version }}
+          name: sift-py-dist-${{ env.PIP_PLATFORM }}-py${{ matrix.python-version }}
           path: python/dist/sift-py-dist-*.zip
           retention-days: 7

From 76b0b8c591bac2c5126cb08ba2f61f3169ee4b01 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:13:20 -0800
Subject: [PATCH 18/75] fix whitespace

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 3fdcd1b6b..25061e926 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -54,7 +54,7 @@ jobs:
 
       - name: Set up QEMU
         if: matrix.arch == 'aarch64'
-       uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
 

From 2edf4ed9f8d3a24c7b2a64e5534d8c431210072f Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:17:52 -0800
Subject: [PATCH 19/75] only use pip platform env var on download

---
 .github/workflows/python_build.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 25061e926..72c68c83f 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -23,7 +23,7 @@ jobs:
     needs: get-python-versions
     runs-on: ${{ matrix.runner }}
     env:
-      PIP_PLATFORM: ${{ matrix.os == 'windows' && format('win_{0}', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
+      DIST_PLATFORM: ${{ matrix.os == 'windows' && format('win_{0}', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
     strategy:
       matrix:
         include:
@@ -78,6 +78,8 @@ jobs:
       - name: Download dependencies
         working-directory: python
         shell: bash
+        env:
+          PIP_PLATFORM: ${{ env.DIST_PLATFORM }}
         run: |
           mkdir -p dist/deps
           # Try to download all packages, preferring wheels but allowing source if wheels aren't available
@@ -129,11 +131,11 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          python -c "import shutil; shutil.make_archive('dist/sift-py-dist-${{ env.PIP_PLATFORM }}-py${{ matrix.python-version }}', 'zip', 'dist')"
+          python -c "import shutil; shutil.make_archive('dist/sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}', 'zip', 'dist')"
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-${{ env.PIP_PLATFORM }}-py${{ matrix.python-version }}
+          name: sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}
           path: python/dist/sift-py-dist-*.zip
           retention-days: 7

From 06c9a1e6025335f22a18b1dc7c0fd200c7297ce3 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:26:08 -0800
Subject: [PATCH 20/75] dont include dev or build extras in offline install
 archive

---
 .github/workflows/python_build.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 72c68c83f..65d2d3884 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -73,7 +73,10 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          pip-compile pyproject.toml --all-extras -o requirements-all.txt
+          # Generate requirements file with only openssl extras
+          # npTDMS doesn't have wheels for all platforms, 
+          # so we need to build it from source which was causing issues on windows
+          pip-compile pyproject.toml --extra=openssl -o requirements-all.txt
 
       - name: Download dependencies
         working-directory: python
@@ -124,8 +127,7 @@ jobs:
           test_install ""  # no extras
           test_install "openssl"
           test_install "build"
-          test_install "dev"
-          test_install "openssl,build,dev"
+          test_install "openssl,build"
 
       - name: Create distribution archive
         working-directory: python

From c72008a2a2790b5d942633dbfaf03be20af8ab4d Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:30:42 -0800
Subject: [PATCH 21/75] fix up test installations step for windows

---
 .github/workflows/python_build.yaml | 42 ++++++++++++++++-------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 65d2d3884..e5df2040c 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -108,26 +108,30 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          # Function to create venv and test installation
-          test_install() {
-            local extras=$1
-            echo "Testing installation with extras: ${extras:-none}"
-            python -m venv test_venv
-            source test_venv/bin/activate || source test_venv/Scripts/activate
-            if [ -z "$extras" ]; then
-              pip install --no-index --find-links=dist/deps dist/*.whl
-            else
-              pip install --no-index --find-links=dist/deps "dist/*.whl[$extras]"
-            fi
-            deactivate
-            rm -rf test_venv
-          }
+          # Get the wheel file name
+          WHEEL_FILE=$(ls dist/*.whl)
+          
+          # Test base installation
+          python -m venv test_venv_base
+          if [ "${{ matrix.os }}" = "windows" ]; then
+            source test_venv_base/Scripts/activate
+          else
+            source test_venv_base/bin/activate
+          fi
+          pip install --no-index --find-links=dist/deps "$WHEEL_FILE"
+          deactivate
+          rm -rf test_venv_base
 
-          # Test each combination in a fresh venv
-          test_install ""  # no extras
-          test_install "openssl"
-          test_install "build"
-          test_install "openssl,build"
+          # Test openssl installation
+          python -m venv test_venv_openssl
+          if [ "${{ matrix.os }}" = "windows" ]; then
+            source test_venv_openssl/Scripts/activate
+          else
+            source test_venv_openssl/bin/activate
+          fi
+          pip install --no-index --find-links=dist/deps "$WHEEL_FILE[openssl]"
+          deactivate
+          rm -rf test_venv_openssl
 
       - name: Create distribution archive
         working-directory: python

From e81a050d44095058572e1aa4468940ed82db3d86 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:37:42 -0800
Subject: [PATCH 22/75] build all desired permutations

---
 .github/workflows/python_build.yaml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index e5df2040c..8ebb299fd 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -19,14 +19,15 @@ jobs:
           echo "versions=$versions" >> $GITHUB_OUTPUT
 
   build_and_verify:
-    name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
+    name: Build and verify on ${{ matrix.config.os }} (${{ matrix.config.arch }}) with Python ${{ matrix.python-version }}
     needs: get-python-versions
-    runs-on: ${{ matrix.runner }}
+    runs-on: ${{ matrix.config.runner }}
     env:
-      DIST_PLATFORM: ${{ matrix.os == 'windows' && format('win_{0}', matrix.arch) || format('{0}-{1}', matrix.os, matrix.arch) }}
+      DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
     strategy:
+      fail-fast: false  # Continue with other builds even if one fails
       matrix:
-        include:
+        config:
           # Ubuntu builds (x86_64 and aarch64)
           - os: ubuntu
             arch: x86_64
@@ -53,7 +54,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        if: matrix.arch == 'aarch64'
+        if: matrix.config.arch == 'aarch64'
         uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
@@ -113,7 +114,7 @@ jobs:
           
           # Test base installation
           python -m venv test_venv_base
-          if [ "${{ matrix.os }}" = "windows" ]; then
+          if [ "${{ matrix.config.os }}" = "windows" ]; then
             source test_venv_base/Scripts/activate
           else
             source test_venv_base/bin/activate
@@ -124,7 +125,7 @@ jobs:
 
           # Test openssl installation
           python -m venv test_venv_openssl
-          if [ "${{ matrix.os }}" = "windows" ]; then
+          if [ "${{ matrix.config.os }}" = "windows" ]; then
             source test_venv_openssl/Scripts/activate
           else
             source test_venv_openssl/bin/activate

From 47b1dc5e384cfb2482d01a73f6dc9befe1e68da3 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:43:27 -0800
Subject: [PATCH 23/75] remove pip platform on download

---
 .github/workflows/python_build.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 8ebb299fd..5ea914e90 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -82,13 +82,10 @@ jobs:
       - name: Download dependencies
         working-directory: python
         shell: bash
-        env:
-          PIP_PLATFORM: ${{ env.DIST_PLATFORM }}
         run: |
           mkdir -p dist/deps
-          # Try to download all packages, preferring wheels but allowing source if wheels aren't available
+          # Download and build packages for the current platform
           pip download -r requirements-all.txt -d dist/deps \
-            --platform $PIP_PLATFORM \
             --python-version ${{ matrix.python-version }} \
             --prefer-binary \
             --no-deps

From 7f7c49a683341fad54197771e2bdd0793cea56b4 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 00:57:42 -0800
Subject: [PATCH 24/75] merge platform distributions

---
 .github/workflows/python_build.yaml | 57 +++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 5ea914e90..a87c5bc42 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -143,3 +143,60 @@ jobs:
           name: sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}
           path: python/dist/sift-py-dist-*.zip
           retention-days: 7
+
+  merge_platform_archives:
+    name: Merge archives for ${{ matrix.config.os }} (${{ matrix.config.arch }})
+    needs: [build_and_verify]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        config:
+          # Ubuntu builds (x86_64 and aarch64)
+          - os: ubuntu
+            arch: x86_64
+          - os: ubuntu
+            arch: aarch64
+            
+          # macOS builds (x86_64 and arm64)
+          - os: macos
+            arch: x86_64
+          - os: macos
+            arch: arm64
+            
+          # Windows builds (x64)
+          - os: windows
+            arch: amd64
+    env:
+      DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
+    steps:
+      - name: Download all Python version archives for platform
+        uses: actions/download-artifact@v4
+        with:
+          pattern: sift-py-dist-${{ env.DIST_PLATFORM }}-py*
+          path: platform_archives
+          merge-multiple: false
+
+      - name: Merge archives
+        shell: bash
+        run: |
+          # Create directory for merged wheels
+          mkdir -p merged/dist/deps
+          
+          # Extract all archives and merge unique wheels
+          for zip in platform_archives/*/sift-py-dist-*.zip; do
+            unzip -o "$zip" -d "temp_extract"
+            # Copy all wheels, letting newer ones overwrite older ones
+            cp -r temp_extract/dist/* merged/dist/
+            rm -rf temp_extract
+          done
+          
+          # Create merged archive
+          cd merged
+          zip -r "../sift-py-dist-${{ env.DIST_PLATFORM }}-all-python.zip" dist/
+
+      - name: Upload merged archive
+        uses: actions/upload-artifact@v4
+        with:
+          name: sift-py-dist-${{ env.DIST_PLATFORM }}-all-python
+          path: sift-py-dist-*.zip
+          retention-days: 7

From 7f42ef66845c1bd16ed87b2b727ada637d63c16b Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 01:04:40 -0800
Subject: [PATCH 25/75] add build deps for macos3.8 and merge platform
 consolidation step

---
 .github/workflows/python_build.yaml | 86 ++++++++++++++---------------
 1 file changed, 43 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index a87c5bc42..0dc638a2f 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -84,7 +84,13 @@ jobs:
         shell: bash
         run: |
           mkdir -p dist/deps
-          # Download and build packages for the current platform
+          # First download build dependencies
+          pip download -d dist/deps \
+            setuptools wheel Cython \
+            --prefer-binary \
+            --no-deps
+
+          # Then download package dependencies
           pip download -r requirements-all.txt -d dist/deps \
             --python-version ${{ matrix.python-version }} \
             --prefer-binary \
@@ -145,58 +151,52 @@ jobs:
           retention-days: 7
 
   merge_platform_archives:
-    name: Merge archives for ${{ matrix.config.os }} (${{ matrix.config.arch }})
+    name: Merge platform archives
     needs: [build_and_verify]
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        config:
-          # Ubuntu builds (x86_64 and aarch64)
-          - os: ubuntu
-            arch: x86_64
-          - os: ubuntu
-            arch: aarch64
-            
-          # macOS builds (x86_64 and arm64)
-          - os: macos
-            arch: x86_64
-          - os: macos
-            arch: arm64
-            
-          # Windows builds (x64)
-          - os: windows
-            arch: amd64
-    env:
-      DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
     steps:
-      - name: Download all Python version archives for platform
+      - name: Download all archives
         uses: actions/download-artifact@v4
         with:
-          pattern: sift-py-dist-${{ env.DIST_PLATFORM }}-py*
+          pattern: sift-py-dist-*-py*
           path: platform_archives
           merge-multiple: false
 
-      - name: Merge archives
+      - name: Merge archives by platform
         shell: bash
         run: |
-          # Create directory for merged wheels
-          mkdir -p merged/dist/deps
-          
-          # Extract all archives and merge unique wheels
-          for zip in platform_archives/*/sift-py-dist-*.zip; do
-            unzip -o "$zip" -d "temp_extract"
-            # Copy all wheels, letting newer ones overwrite older ones
-            cp -r temp_extract/dist/* merged/dist/
-            rm -rf temp_extract
-          done
-          
-          # Create merged archive
-          cd merged
-          zip -r "../sift-py-dist-${{ env.DIST_PLATFORM }}-all-python.zip" dist/
-
-      - name: Upload merged archive
+          # Function to merge archives for a platform
+          merge_platform_archives() {
+            local platform=$1
+            echo "Merging archives for platform: $platform"
+            
+            # Create directory for merged wheels
+            mkdir -p "merged_$platform/dist/deps"
+            
+            # Extract all archives and merge unique wheels
+            for zip in platform_archives/*/sift-py-dist-$platform-py*.zip; do
+              unzip -o "$zip" -d "temp_extract"
+              # Copy all wheels, letting newer ones overwrite older ones
+              cp -r temp_extract/dist/* "merged_$platform/dist/"
+              rm -rf temp_extract
+            done
+            
+            # Create merged archive
+            cd "merged_$platform"
+            zip -r "../sift-py-dist-$platform-all-python.zip" dist/
+            cd ..
+          }
+
+          # Merge archives for each platform
+          merge_platform_archives "ubuntu-x86_64"
+          merge_platform_archives "ubuntu-aarch64"
+          merge_platform_archives "macos-x86_64"
+          merge_platform_archives "macos-arm64"
+          merge_platform_archives "win_amd64"
+
+      - name: Upload merged archives
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-${{ env.DIST_PLATFORM }}-all-python
-          path: sift-py-dist-*.zip
+          name: sift-py-dist-all-platforms
+          path: sift-py-dist-*-all-python.zip
           retention-days: 7

From fb125a5edce9ab505a01b971da37eda6486a4420 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 01:14:11 -0800
Subject: [PATCH 26/75] use zip instead of python shutil.make_archive

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 0dc638a2f..8493b719a 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -141,7 +141,7 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          python -c "import shutil; shutil.make_archive('dist/sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}', 'zip', 'dist')"
+          cd dist && zip -r "sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}.zip" *
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v4

From 2929030978ad03ba591278ea4c8fda32cc7a3248 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 01:18:12 -0800
Subject: [PATCH 27/75] use powershell compress archive on windows

---
 .github/workflows/python_build.yaml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 8493b719a..d42d72297 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -141,7 +141,13 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          cd dist && zip -r "sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}.zip" *
+          if [ "${{ matrix.config.os }}" = "windows" ]; then
+            # Use PowerShell's Compress-Archive on Windows
+            pwsh -Command "Compress-Archive -Path dist/* -DestinationPath dist/sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}.zip -Force"
+          else
+            # Use zip on Unix-like systems
+            cd dist && zip -r "sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}.zip" *
+          fi
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v4

From 3fa50c876e9911dfd2d4087d08a01dda72b4d18a Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 01:23:06 -0800
Subject: [PATCH 28/75] debug

---
 .github/workflows/python_build.yaml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d42d72297..32cce25a4 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -181,12 +181,27 @@ jobs:
             
             # Extract all archives and merge unique wheels
             for zip in platform_archives/*/sift-py-dist-$platform-py*.zip; do
+              echo "Processing archive: $zip"
+              
+              # Create fresh temp directory
+              rm -rf temp_extract
+              mkdir -p temp_extract
+              
+              # Extract and show contents
               unzip -o "$zip" -d "temp_extract"
-              # Copy all wheels, letting newer ones overwrite older ones
-              cp -r temp_extract/dist/* "merged_$platform/dist/"
+              echo "Contents of temp_extract:"
+              ls -R temp_extract
+              
+              # Find and copy all files from dist directory
+              find temp_extract -type f -exec cp {} "merged_$platform/dist/" \;
+              
+              # Cleanup
               rm -rf temp_extract
             done
             
+            echo "Contents of merged directory:"
+            ls -R "merged_$platform/dist/"
+            
             # Create merged archive
             cd "merged_$platform"
             zip -r "../sift-py-dist-$platform-all-python.zip" dist/

From 06864000c09e4bd08ca9440bf767e5d2bff8adf9 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 01:30:50 -0800
Subject: [PATCH 29/75] upload as separate files

---
 .github/workflows/python_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 32cce25a4..ccfdef6cf 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -218,6 +218,7 @@ jobs:
       - name: Upload merged archives
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-all-platforms
+          name: sift-py-dist-platform-archives
           path: sift-py-dist-*-all-python.zip
           retention-days: 7
+          separate-files: true

From 95367fb9b4d80ff5345e1f91bf4dd2825c602e57 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 01:54:51 -0800
Subject: [PATCH 30/75] matrix the merge job

---
 .github/workflows/python_build.yaml | 78 ++++++++++-------------------
 1 file changed, 27 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index ccfdef6cf..d96c32cba 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -49,6 +49,8 @@ jobs:
             arch: amd64
             runner: windows-latest
         python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
+    outputs:
+      platforms: ${{ toJson(matrix.config) }}
 
     steps:
       - uses: actions/checkout@v4
@@ -157,68 +159,42 @@ jobs:
           retention-days: 7
 
   merge_platform_archives:
-    name: Merge platform archives
+    name: Merge archives for ${{ matrix.config.os }} (${{ matrix.config.arch }})
     needs: [build_and_verify]
     runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJson(needs.build_and_verify.outputs.platforms) }}
+    env:
+      DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
     steps:
-      - name: Download all archives
+      - name: Download platform archives
         uses: actions/download-artifact@v4
         with:
-          pattern: sift-py-dist-*-py*
+          pattern: sift-py-dist-${{ env.DIST_PLATFORM }}-py*
           path: platform_archives
           merge-multiple: false
 
-      - name: Merge archives by platform
+      - name: Merge archives
         shell: bash
         run: |
-          # Function to merge archives for a platform
-          merge_platform_archives() {
-            local platform=$1
-            echo "Merging archives for platform: $platform"
-            
-            # Create directory for merged wheels
-            mkdir -p "merged_$platform/dist/deps"
-            
-            # Extract all archives and merge unique wheels
-            for zip in platform_archives/*/sift-py-dist-$platform-py*.zip; do
-              echo "Processing archive: $zip"
-              
-              # Create fresh temp directory
-              rm -rf temp_extract
-              mkdir -p temp_extract
-              
-              # Extract and show contents
-              unzip -o "$zip" -d "temp_extract"
-              echo "Contents of temp_extract:"
-              ls -R temp_extract
-              
-              # Find and copy all files from dist directory
-              find temp_extract -type f -exec cp {} "merged_$platform/dist/" \;
-              
-              # Cleanup
-              rm -rf temp_extract
-            done
-            
-            echo "Contents of merged directory:"
-            ls -R "merged_$platform/dist/"
-            
-            # Create merged archive
-            cd "merged_$platform"
-            zip -r "../sift-py-dist-$platform-all-python.zip" dist/
-            cd ..
-          }
-
-          # Merge archives for each platform
-          merge_platform_archives "ubuntu-x86_64"
-          merge_platform_archives "ubuntu-aarch64"
-          merge_platform_archives "macos-x86_64"
-          merge_platform_archives "macos-arm64"
-          merge_platform_archives "win_amd64"
-
-      - name: Upload merged archives
+          # Create directory for merged wheels
+          mkdir -p merged/dist/deps
+          
+          # Extract and merge all archives for this platform
+          for zip in platform_archives/*/sift-py-dist-${{ env.DIST_PLATFORM }}-py*.zip; do
+            echo "Processing archive: $zip"
+            unzip -o "$zip" -d "temp_extract"
+            cp -r temp_extract/dist/* merged/dist/
+            rm -rf temp_extract
+          done
+          
+          # Create merged archive
+          cd merged
+          zip -r "../sift-py-dist-${{ env.DIST_PLATFORM }}-all-python.zip" dist/
+
+      - name: Upload merged archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-platform-archives
+          name: sift-py-dist-${{ env.DIST_PLATFORM }}-all-python
           path: sift-py-dist-*-all-python.zip
           retention-days: 7
-          separate-files: true

From e5a50af288cb3ff6615eb087914415c545933900 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 01:59:04 -0800
Subject: [PATCH 31/75] fix up

---
 .github/workflows/python_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d96c32cba..cbd9b4af6 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -163,7 +163,8 @@ jobs:
     needs: [build_and_verify]
     runs-on: ubuntu-latest
     strategy:
-      matrix: ${{ fromJson(needs.build_and_verify.outputs.platforms) }}
+      matrix:
+        config: ${{ fromJson(needs.build_and_verify.outputs.platforms) }}
     env:
       DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
     steps:

From b53e03a800dffe35712d9b3d7be2b75f01ab8d68 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 02:02:55 -0800
Subject: [PATCH 32/75] Update GitHub Actions workflow to correctly output
 platform configurations using matrix expansion

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index cbd9b4af6..c9e70a04b 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -50,7 +50,7 @@ jobs:
             runner: windows-latest
         python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
     outputs:
-      platforms: ${{ toJson(matrix.config) }}
+      platforms: ${{ toJson(matrix.config.*) }}
 
     steps:
       - uses: actions/checkout@v4

From ee4ae1e69ef1eca059762255ee328d8c93acffb4 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 02:16:09 -0800
Subject: [PATCH 33/75] Enhance GitHub Actions workflow by defining platform
 configurations in a separate job, streamlining matrix management for build
 and merge processes. This change improves clarity and maintainability of the
 CI/CD pipeline.

---
 .github/workflows/python_build.yaml | 57 ++++++++++++++---------------
 1 file changed, 28 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index c9e70a04b..3dfb9d631 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -18,6 +18,27 @@ jobs:
           versions=$(grep "Programming Language :: Python :: " python/pyproject.toml | sed 's/.*Python :: \([0-9.]*\).*/\1/' | jq -R -s -c 'split("\n")[:-1]')
           echo "versions=$versions" >> $GITHUB_OUTPUT
 
+  define-platforms:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          matrix=$(cat << 'EOF'
+          {
+            "config": [
+              {"os": "ubuntu", "arch": "x86_64", "runner": "ubuntu-latest"},
+              {"os": "ubuntu", "arch": "aarch64", "runner": "ubuntu-latest"},
+              {"os": "macos", "arch": "x86_64", "runner": "macos-latest"},
+              {"os": "macos", "arch": "arm64", "runner": "macos-14"},
+              {"os": "windows", "arch": "amd64", "runner": "windows-latest"}
+            ]
+          }
+          EOF
+          )
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
   build_and_verify:
     name: Build and verify on ${{ matrix.config.os }} (${{ matrix.config.arch }}) with Python ${{ matrix.python-version }}
     needs: get-python-versions
@@ -26,31 +47,9 @@ jobs:
       DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
     strategy:
       fail-fast: false  # Continue with other builds even if one fails
-      matrix:
-        config:
-          # Ubuntu builds (x86_64 and aarch64)
-          - os: ubuntu
-            arch: x86_64
-            runner: ubuntu-latest
-          - os: ubuntu
-            arch: aarch64
-            runner: ubuntu-latest
-            
-          # macOS builds (x86_64 and arm64)
-          - os: macos
-            arch: x86_64
-            runner: macos-latest
-          - os: macos
-            arch: arm64
-            runner: macos-14
-            
-          # Windows builds (x64)
-          - os: windows
-            arch: amd64
-            runner: windows-latest
-        python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
+      matrix: ${{ fromJson(needs.define-platforms.outputs.matrix) }}
     outputs:
-      platforms: ${{ toJson(matrix.config.*) }}
+      matrix-config: ${{ toJson(matrix.config) }}
 
     steps:
       - uses: actions/checkout@v4
@@ -159,14 +158,14 @@ jobs:
           retention-days: 7
 
   merge_platform_archives:
-    name: Merge archives for ${{ matrix.config.os }} (${{ matrix.config.arch }})
-    needs: [build_and_verify]
+    name: Merge archives for ${{ matrix.platform.os }} (${{ matrix.platform.arch }})
+    needs: [build_and_verify, define-platforms]
     runs-on: ubuntu-latest
     strategy:
-      matrix:
-        config: ${{ fromJson(needs.build_and_verify.outputs.platforms) }}
+      matrix: 
+        platform: ${{ fromJson(needs.define-platforms.outputs.matrix).config }}
     env:
-      DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
+      DIST_PLATFORM: ${{ matrix.platform.os == 'windows' && format('win_{0}', matrix.platform.arch) || format('{0}-{1}', matrix.platform.os, matrix.platform.arch) }}
     steps:
       - name: Download platform archives
         uses: actions/download-artifact@v4

From 50652f26270a5e9d20bb500873ce0db51aafcb7e Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 02:18:22 -0800
Subject: [PATCH 34/75] fixup

---
 .github/workflows/python_build.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 3dfb9d631..58bc5771d 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -41,13 +41,15 @@ jobs:
 
   build_and_verify:
     name: Build and verify on ${{ matrix.config.os }} (${{ matrix.config.arch }}) with Python ${{ matrix.python-version }}
-    needs: get-python-versions
+    needs: [get-python-versions, define-platforms]
     runs-on: ${{ matrix.config.runner }}
     env:
       DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
     strategy:
       fail-fast: false  # Continue with other builds even if one fails
-      matrix: ${{ fromJson(needs.define-platforms.outputs.matrix) }}
+      matrix:
+        config: ${{ fromJson(needs.define-platforms.outputs.matrix).config }}
+        python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
     outputs:
       matrix-config: ${{ toJson(matrix.config) }}
 

From 5d08c023589ee8e093faf782c83dd3150ce7abcb Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 02:20:45 -0800
Subject: [PATCH 35/75] try this

---
 .github/workflows/python_build.yaml | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 58bc5771d..cd348eb78 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -24,20 +24,19 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - id: set-matrix
-        run: |
-          matrix=$(cat << 'EOF'
-          {
-            "config": [
-              {"os": "ubuntu", "arch": "x86_64", "runner": "ubuntu-latest"},
-              {"os": "ubuntu", "arch": "aarch64", "runner": "ubuntu-latest"},
-              {"os": "macos", "arch": "x86_64", "runner": "macos-latest"},
-              {"os": "macos", "arch": "arm64", "runner": "macos-14"},
-              {"os": "windows", "arch": "amd64", "runner": "windows-latest"}
-            ]
-          }
-          EOF
-          )
-          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const matrix = {
+              config: [
+                {os: 'ubuntu', arch: 'x86_64', runner: 'ubuntu-latest'},
+                {os: 'ubuntu', arch: 'aarch64', runner: 'ubuntu-latest'},
+                {os: 'macos', arch: 'x86_64', runner: 'macos-latest'},
+                {os: 'macos', arch: 'arm64', runner: 'macos-14'},
+                {os: 'windows', arch: 'amd64', runner: 'windows-latest'}
+              ]
+            };
+            core.setOutput('matrix', JSON.stringify(matrix));
 
   build_and_verify:
     name: Build and verify on ${{ matrix.config.os }} (${{ matrix.config.arch }}) with Python ${{ matrix.python-version }}

From d1914118a622eabe9b25466eda6342085c65e16b Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 02:24:38 -0800
Subject: [PATCH 36/75] Refactor GitHub Actions workflow to consolidate
 platform and Python version retrieval into a single job, enhancing clarity
 and maintainability. Update build and merge jobs to utilize the new matrix
 configuration outputs, ensuring consistent platform handling across the CI/CD
 pipeline.

---
 .github/workflows/python_build.yaml | 41 +++++++++++++++++++----------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index cd348eb78..7c87bca62 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -5,10 +5,12 @@ on:
   pull_request:
 
 jobs:
-  get-python-versions:
+  get-matrix-config:
+    name: Get matrix configuration
     runs-on: ubuntu-latest
     outputs:
       python-versions: ${{ steps.get-versions.outputs.versions }}
+      platforms: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
       
@@ -18,12 +20,8 @@ jobs:
           versions=$(grep "Programming Language :: Python :: " python/pyproject.toml | sed 's/.*Python :: \([0-9.]*\).*/\1/' | jq -R -s -c 'split("\n")[:-1]')
           echo "versions=$versions" >> $GITHUB_OUTPUT
 
-  define-platforms:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - id: set-matrix
+      - name: Set platform matrix
+        id: set-matrix
         uses: actions/github-script@v7
         with:
           script: |
@@ -40,15 +38,15 @@ jobs:
 
   build_and_verify:
     name: Build and verify on ${{ matrix.config.os }} (${{ matrix.config.arch }}) with Python ${{ matrix.python-version }}
-    needs: [get-python-versions, define-platforms]
+    needs: get-matrix-config
     runs-on: ${{ matrix.config.runner }}
     env:
       DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
     strategy:
-      fail-fast: false  # Continue with other builds even if one fails
+      fail-fast: false
       matrix:
-        config: ${{ fromJson(needs.define-platforms.outputs.matrix).config }}
-        python-version: ${{fromJson(needs.get-python-versions.outputs.python-versions)}}
+        config: ${{ fromJson(needs.get-matrix-config.outputs.platforms).config }}
+        python-version: ${{fromJson(needs.get-matrix-config.outputs.python-versions)}}
     outputs:
       matrix-config: ${{ toJson(matrix.config) }}
 
@@ -160,11 +158,11 @@ jobs:
 
   merge_platform_archives:
     name: Merge archives for ${{ matrix.platform.os }} (${{ matrix.platform.arch }})
-    needs: [build_and_verify, define-platforms]
+    needs: [build_and_verify, get-matrix-config]
     runs-on: ubuntu-latest
     strategy:
       matrix: 
-        platform: ${{ fromJson(needs.define-platforms.outputs.matrix).config }}
+        platform: ${{ fromJson(needs.get-matrix-config.outputs.platforms).config }}
     env:
       DIST_PLATFORM: ${{ matrix.platform.os == 'windows' && format('win_{0}', matrix.platform.arch) || format('{0}-{1}', matrix.platform.os, matrix.platform.arch) }}
     steps:
@@ -184,11 +182,26 @@ jobs:
           # Extract and merge all archives for this platform
           for zip in platform_archives/*/sift-py-dist-${{ env.DIST_PLATFORM }}-py*.zip; do
             echo "Processing archive: $zip"
+            
+            # Create fresh temp directory
+            rm -rf temp_extract
+            mkdir -p temp_extract
+            
+            # Extract and show contents
             unzip -o "$zip" -d "temp_extract"
-            cp -r temp_extract/dist/* merged/dist/
+            echo "Contents of temp_extract:"
+            ls -R temp_extract
+            
+            # Find all files recursively and copy them
+            find temp_extract -type f -exec cp {} merged/dist/ \;
+            
+            # Cleanup
             rm -rf temp_extract
           done
           
+          echo "Contents of merged directory:"
+          ls -R merged/dist/
+          
           # Create merged archive
           cd merged
           zip -r "../sift-py-dist-${{ env.DIST_PLATFORM }}-all-python.zip" dist/

From c3f67628df003875f3a8d7eae21d956d1aa6a079 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 02:33:24 -0800
Subject: [PATCH 37/75] remove qemu since we arent doing cross comp

---
 .github/workflows/python_build.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 7c87bca62..9fb7312bf 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -53,12 +53,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        if: matrix.config.arch == 'aarch64'
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: arm64
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

From a049d95ad6e39f476391e711eb790680719e643c Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:14:47 -0800
Subject: [PATCH 38/75] some more cleanup

---
 .github/workflows/python_build.yaml | 64 ++++++++++++++---------------
 1 file changed, 31 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 9fb7312bf..c0b4cb953 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -20,35 +20,33 @@ jobs:
           versions=$(grep "Programming Language :: Python :: " python/pyproject.toml | sed 's/.*Python :: \([0-9.]*\).*/\1/' | jq -R -s -c 'split("\n")[:-1]')
           echo "versions=$versions" >> $GITHUB_OUTPUT
 
+      # We define the platforms here so we can reuse the matrix in multiple jobs
+      # This is a workaround for the fact that we can't use the same matrix in multiple jobs
       - name: Set platform matrix
         id: set-matrix
         uses: actions/github-script@v7
         with:
           script: |
-            const matrix = {
-              config: [
-                {os: 'ubuntu', arch: 'x86_64', runner: 'ubuntu-latest'},
-                {os: 'ubuntu', arch: 'aarch64', runner: 'ubuntu-latest'},
-                {os: 'macos', arch: 'x86_64', runner: 'macos-latest'},
-                {os: 'macos', arch: 'arm64', runner: 'macos-14'},
-                {os: 'windows', arch: 'amd64', runner: 'windows-latest'}
-              ]
-            };
+            const matrix = [
+              {os: 'ubuntu', arch: 'x86_64', runner: 'ubuntu-latest', platform: 'ubuntu-x86_64'},
+              {os: 'ubuntu', arch: 'aarch64', runner: 'ubuntu-latest', platform: 'ubuntu-aarch64'},
+              {os: 'macos', arch: 'x86_64', runner: 'macos-latest', platform: 'macos-x86_64'},
+              {os: 'macos', arch: 'arm64', runner: 'macos-14', platform: 'macos-arm64'},
+              {os: 'windows', arch: 'amd64', runner: 'windows-latest', platform: 'win_amd64'}
+            ];
             core.setOutput('matrix', JSON.stringify(matrix));
 
   build_and_verify:
-    name: Build and verify on ${{ matrix.config.os }} (${{ matrix.config.arch }}) with Python ${{ matrix.python-version }}
+    name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-matrix-config
-    runs-on: ${{ matrix.config.runner }}
-    env:
-      DIST_PLATFORM: ${{ matrix.config.os == 'windows' && format('win_{0}', matrix.config.arch) || format('{0}-{1}', matrix.config.os, matrix.config.arch) }}
+    runs-on: ${{ matrix.runner }}
     strategy:
       fail-fast: false
       matrix:
-        config: ${{ fromJson(needs.get-matrix-config.outputs.platforms).config }}
+        platform: ${{ fromJson(needs.get-matrix-config.outputs.platforms) }}
         python-version: ${{fromJson(needs.get-matrix-config.outputs.python-versions)}}
     outputs:
-      matrix-config: ${{ toJson(matrix.config) }}
+      matrix-config: ${{ toJson(matrix) }}
 
     steps:
       - uses: actions/checkout@v4
@@ -111,7 +109,7 @@ jobs:
           
           # Test base installation
           python -m venv test_venv_base
-          if [ "${{ matrix.config.os }}" = "windows" ]; then
+          if [ "${{ matrix.os }}" = "windows" ]; then
             source test_venv_base/Scripts/activate
           else
             source test_venv_base/bin/activate
@@ -122,7 +120,7 @@ jobs:
 
           # Test openssl installation
           python -m venv test_venv_openssl
-          if [ "${{ matrix.config.os }}" = "windows" ]; then
+          if [ "${{ matrix.os }}" = "windows" ]; then
             source test_venv_openssl/Scripts/activate
           else
             source test_venv_openssl/bin/activate
@@ -135,37 +133,37 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          if [ "${{ matrix.config.os }}" = "windows" ]; then
-            # Use PowerShell's Compress-Archive on Windows
-            pwsh -Command "Compress-Archive -Path dist/* -DestinationPath dist/sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}.zip -Force"
+          if [ "${{ matrix.os }}" = "windows" ]; then
+            pwsh -Command "Compress-Archive -Path dist/* -DestinationPath dist/sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform }}.zip -Force"
           else
-            # Use zip on Unix-like systems
-            cd dist && zip -r "sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}.zip" *
+            cd dist && zip -r "sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform }}.zip" *
           fi
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-${{ env.DIST_PLATFORM }}-py${{ matrix.python-version }}
+          name: sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform }}
           path: python/dist/sift-py-dist-*.zip
           retention-days: 7
 
   merge_platform_archives:
-    name: Merge archives for ${{ matrix.platform.os }} (${{ matrix.platform.arch }})
+    name: Merge archives for ${{ matrix.os }} (${{ matrix.arch }})
     needs: [build_and_verify, get-matrix-config]
     runs-on: ubuntu-latest
     strategy:
-      matrix: 
-        platform: ${{ fromJson(needs.get-matrix-config.outputs.platforms).config }}
-    env:
-      DIST_PLATFORM: ${{ matrix.platform.os == 'windows' && format('win_{0}', matrix.platform.arch) || format('{0}-{1}', matrix.platform.os, matrix.platform.arch) }}
+      matrix: ${{ fromJson(needs.get-matrix-config.outputs.platforms) }}
     steps:
       - name: Download platform archives
         uses: actions/download-artifact@v4
         with:
-          pattern: sift-py-dist-${{ env.DIST_PLATFORM }}-py*
+          pattern: sift-py-dist-py*-${{ matrix.platform }}
           path: platform_archives
           merge-multiple: false
+    
+      - name: debug 
+        run: |
+          echo "{{ needs.build_and_verify.outputs.matrix-config }}"
+          python -c "import sysconfig;import re;print(re.sub(r'[-.]', '_', sysconfig.get_platform()))"
 
       - name: Merge archives
         shell: bash
@@ -174,7 +172,7 @@ jobs:
           mkdir -p merged/dist/deps
           
           # Extract and merge all archives for this platform
-          for zip in platform_archives/*/sift-py-dist-${{ env.DIST_PLATFORM }}-py*.zip; do
+          for zip in platform_archives/*/sift-py-dist-py*-${{ matrix.platform }}.zip; do
             echo "Processing archive: $zip"
             
             # Create fresh temp directory
@@ -198,11 +196,11 @@ jobs:
           
           # Create merged archive
           cd merged
-          zip -r "../sift-py-dist-${{ env.DIST_PLATFORM }}-all-python.zip" dist/
+          zip -r "../sift-py-dist-py3-${{ matrix.platform }}.zip" dist/
 
       - name: Upload merged archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-${{ env.DIST_PLATFORM }}-all-python
-          path: sift-py-dist-*-all-python.zip
+          name: sift-py-dist-py3-${{ matrix.platform }}
+          path: sift-py-dist-py3-*.zip
           retention-days: 7

From e1c448b3ee9e82af9abf99b1f32a3ed8cef2727d Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:18:43 -0800
Subject: [PATCH 39/75] fixup

---
 .github/workflows/python_build.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index c0b4cb953..7761b4ddc 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -39,7 +39,7 @@ jobs:
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-matrix-config
-    runs-on: ${{ matrix.runner }}
+    runs-on: ${{ needs.get-matrix-config.outputs.platforms.runner }}
     strategy:
       fail-fast: false
       matrix:
@@ -62,6 +62,10 @@ jobs:
           python -m pip install --upgrade pip
           pip install build pip-tools twine
 
+      - name: debug 
+        run: |
+          python -c "import sysconfig;import re;print(re.sub(r'[-.]', '_', sysconfig.get_platform()))"
+
       - name: Generate requirements
         working-directory: python
         shell: bash

From e301182489b26a341d4970e8be64a10e2390b287 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:23:38 -0800
Subject: [PATCH 40/75] try again

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 7761b4ddc..b3a7d165e 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -39,7 +39,7 @@ jobs:
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-matrix-config
-    runs-on: ${{ needs.get-matrix-config.outputs.platforms.runner }}
+    runs-on: ${{ fromJson(needs.get-matrix-config.outputs.platforms).runner }}
     strategy:
       fail-fast: false
       matrix:

From b1f9c7dd9f7b2822ebcf11088a47edf33094b39c Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:27:35 -0800
Subject: [PATCH 41/75] more debug

---
 .github/workflows/python_build.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index b3a7d165e..be573c9f0 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -35,11 +35,15 @@ jobs:
               {os: 'windows', arch: 'amd64', runner: 'windows-latest', platform: 'win_amd64'}
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
+    - run: |
+        echo "{{ needs.get-matrix-config.outputs.platforms }}" ||
+        echo "{{ fromJson(needs.get-matrix-config.outputs.platforms) }}" || 
+        echo "{{ fromJson(needs.get-matrix-config.outputs.platforms).runner }}" 
 
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-matrix-config
-    runs-on: ${{ fromJson(needs.get-matrix-config.outputs.platforms).runner }}
+    runs-on: ${{ fromJson(needs.get-matrix-config.outputs.platforms).runner) }}
     strategy:
       fail-fast: false
       matrix:

From bdab5dce30098b8f8dec8dc816760843dc7ccfa3 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:28:33 -0800
Subject: [PATCH 42/75] debug

---
 .github/workflows/python_build.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index be573c9f0..d5b20666a 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -36,9 +36,9 @@ jobs:
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
     - run: |
-        echo "{{ needs.get-matrix-config.outputs.platforms }}" ||
-        echo "{{ fromJson(needs.get-matrix-config.outputs.platforms) }}" || 
-        echo "{{ fromJson(needs.get-matrix-config.outputs.platforms).runner }}" 
+        echo "{{ steps.set-matrix.outputs.matrix }}" ||
+        echo "{{ fromJson(steps.set-matrix.outputs.matrix) }}" || 
+        echo "{{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
 
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}

From da755656ec06b0e050574d786f62eaba6661f6ce Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:29:12 -0800
Subject: [PATCH 43/75] fixup

---
 .github/workflows/python_build.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d5b20666a..37cd05e85 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -35,10 +35,11 @@ jobs:
               {os: 'windows', arch: 'amd64', runner: 'windows-latest', platform: 'win_amd64'}
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
-    - run: |
-        echo "{{ steps.set-matrix.outputs.matrix }}" ||
-        echo "{{ fromJson(steps.set-matrix.outputs.matrix) }}" || 
-        echo "{{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
+      - name: debug 
+        run: |
+          echo "{{ steps.set-matrix.outputs.matrix }}" ||
+          echo "{{ fromJson(steps.set-matrix.outputs.matrix) }}" || 
+          echo "{{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
 
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}

From 364b615254bd8af05db387cc28c9cedcb56fdfca Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:33:32 -0800
Subject: [PATCH 44/75] ugh

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 37cd05e85..379e6e8dd 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -44,7 +44,7 @@ jobs:
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-matrix-config
-    runs-on: ${{ fromJson(needs.get-matrix-config.outputs.platforms).runner) }}
+    runs-on: ${{ fromJson(needs.get-matrix-config.outputs.platforms).runner }}
     strategy:
       fail-fast: false
       matrix:

From e128b2febb60ec4ef96771776888b4ce7e7e8860 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:35:41 -0800
Subject: [PATCH 45/75] seriously

---
 .github/workflows/python_build.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 379e6e8dd..9c7e08470 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -37,9 +37,9 @@ jobs:
             core.setOutput('matrix', JSON.stringify(matrix));
       - name: debug 
         run: |
-          echo "{{ steps.set-matrix.outputs.matrix }}" ||
-          echo "{{ fromJson(steps.set-matrix.outputs.matrix) }}" || 
-          echo "{{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
+          echo "${{ steps.set-matrix.outputs.matrix }}" ||
+          echo "${{ fromJson(steps.set-matrix.outputs.matrix) }}" || 
+          echo "${{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
 
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}

From 73e79bd849ccf6712b6235bbf3342b0567b75921 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:37:56 -0800
Subject: [PATCH 46/75] try again

---
 .github/workflows/python_build.yaml | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 9c7e08470..dafc63d0c 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -36,9 +36,17 @@ jobs:
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
       - name: debug 
+        always: true
         run: |
-          echo "${{ steps.set-matrix.outputs.matrix }}" ||
-          echo "${{ fromJson(steps.set-matrix.outputs.matrix) }}" || 
+          echo "${{ steps.set-matrix.outputs.matrix }}"
+            - name: debug 
+      - name: debug 2
+        always: true
+        run: |
+          echo "${{ fromJson(steps.set-matrix.outputs.matrix) }}" 
+      - name: debug 3
+        always: true
+        run: |  
           echo "${{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
 
   build_and_verify:

From 9c1c9583e011118d8dfe5ea1914bef3a87b92faa Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:38:54 -0800
Subject: [PATCH 47/75] here we goooo

---
 .github/workflows/python_build.yaml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index dafc63d0c..b8e1a2ae2 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -36,16 +36,15 @@ jobs:
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
       - name: debug 
-        always: true
+        if: always()
         run: |
           echo "${{ steps.set-matrix.outputs.matrix }}"
-            - name: debug 
       - name: debug 2
-        always: true
+        if: always()
         run: |
           echo "${{ fromJson(steps.set-matrix.outputs.matrix) }}" 
       - name: debug 3
-        always: true
+        if: always()
         run: |  
           echo "${{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
 

From 97c1104a33261dfb7254c88795d8982f9a6f7c64 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:41:55 -0800
Subject: [PATCH 48/75] maybe

---
 .github/workflows/python_build.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index b8e1a2ae2..51887ecf8 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -42,11 +42,11 @@ jobs:
       - name: debug 2
         if: always()
         run: |
-          echo "${{ fromJson(steps.set-matrix.outputs.matrix) }}" 
+          echo "${{ toJson(steps.set-matrix.outputs.matrix) }}" 
       - name: debug 3
         if: always()
         run: |  
-          echo "${{ fromJson(needs.get-matrix-config.outputs.matrix).runner }}" 
+          echo "${{ toJson(needs.get-matrix-config.outputs.matrix).runner }}" 
 
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}

From 3b0ad6200f173ecd2b4050a0ccff46e7057d56d8 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:44:51 -0800
Subject: [PATCH 49/75] lets goo

---
 .github/workflows/python_build.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 51887ecf8..8b0a1d02c 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -42,16 +42,16 @@ jobs:
       - name: debug 2
         if: always()
         run: |
-          echo "${{ toJson(steps.set-matrix.outputs.matrix) }}" 
+          echo "${{ fromJson(steps.set-matrix.outputs.matrix)[0].runner }}" 
       - name: debug 3
         if: always()
         run: |  
-          echo "${{ toJson(needs.get-matrix-config.outputs.matrix).runner }}" 
+          echo "${{ needs.get-matrix-config.outputs.matrix[0].runner }}" 
 
   build_and_verify:
     name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
     needs: get-matrix-config
-    runs-on: ${{ fromJson(needs.get-matrix-config.outputs.platforms).runner }}
+    runs-on: ${{ matrix.platform.runner }}
     strategy:
       fail-fast: false
       matrix:

From 6c7e086c46ccf6563457045029a25a71b9f2099e Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:51:42 -0800
Subject: [PATCH 50/75] ok i think we have this

---
 .github/workflows/python_build.yaml | 34 ++++++++++++++---------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 8b0a1d02c..0322243e9 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -28,11 +28,11 @@ jobs:
         with:
           script: |
             const matrix = [
-              {os: 'ubuntu', arch: 'x86_64', runner: 'ubuntu-latest', platform: 'ubuntu-x86_64'},
-              {os: 'ubuntu', arch: 'aarch64', runner: 'ubuntu-latest', platform: 'ubuntu-aarch64'},
-              {os: 'macos', arch: 'x86_64', runner: 'macos-latest', platform: 'macos-x86_64'},
-              {os: 'macos', arch: 'arm64', runner: 'macos-14', platform: 'macos-arm64'},
-              {os: 'windows', arch: 'amd64', runner: 'windows-latest', platform: 'win_amd64'}
+              {os: 'ubuntu', arch: 'x86_64', runner: 'ubuntu-latest', platform_tag: 'ubuntu-x86_64'},
+              {os: 'ubuntu', arch: 'aarch64', runner: 'ubuntu-latest', platform_tag: 'ubuntu-aarch64'},
+              {os: 'macos', arch: 'x86_64', runner: 'macos-latest', platform_tag: 'macos-x86_64'},
+              {os: 'macos', arch: 'arm64', runner: 'macos-14', platform_tag: 'macos-arm64'},
+              {os: 'windows', arch: 'amd64', runner: 'windows-latest', platform_tag: 'win_amd64'}
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
       - name: debug 
@@ -49,7 +49,7 @@ jobs:
           echo "${{ needs.get-matrix-config.outputs.matrix[0].runner }}" 
 
   build_and_verify:
-    name: Build and verify on ${{ matrix.os }} (${{ matrix.arch }}) with Python ${{ matrix.python-version }}
+    name: Build and verify on ${{ matrix.platform.os }} (${{ matrix.platform.arch }}) with Python ${{ matrix.python-version }}
     needs: get-matrix-config
     runs-on: ${{ matrix.platform.runner }}
     strategy:
@@ -125,7 +125,7 @@ jobs:
           
           # Test base installation
           python -m venv test_venv_base
-          if [ "${{ matrix.os }}" = "windows" ]; then
+          if [ "${{ matrix.platform.os }}" = "windows" ]; then
             source test_venv_base/Scripts/activate
           else
             source test_venv_base/bin/activate
@@ -136,7 +136,7 @@ jobs:
 
           # Test openssl installation
           python -m venv test_venv_openssl
-          if [ "${{ matrix.os }}" = "windows" ]; then
+          if [ "${{ matrix.platform.os }}" = "windows" ]; then
             source test_venv_openssl/Scripts/activate
           else
             source test_venv_openssl/bin/activate
@@ -149,21 +149,21 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          if [ "${{ matrix.os }}" = "windows" ]; then
-            pwsh -Command "Compress-Archive -Path dist/* -DestinationPath dist/sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform }}.zip -Force"
+          if [ "${{ matrix.platform.os }}" = "windows" ]; then
+            pwsh -Command "Compress-Archive -Path dist/* -DestinationPath dist/sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}.zip -Force"
           else
-            cd dist && zip -r "sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform }}.zip" *
+            cd dist && zip -r "sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}.zip" *
           fi
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform }}
+          name: sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}
           path: python/dist/sift-py-dist-*.zip
           retention-days: 7
 
   merge_platform_archives:
-    name: Merge archives for ${{ matrix.os }} (${{ matrix.arch }})
+    name: Merge archives for ${{ matrix.platform.os }} (${{ matrix.platform.arch }})
     needs: [build_and_verify, get-matrix-config]
     runs-on: ubuntu-latest
     strategy:
@@ -172,7 +172,7 @@ jobs:
       - name: Download platform archives
         uses: actions/download-artifact@v4
         with:
-          pattern: sift-py-dist-py*-${{ matrix.platform }}
+          pattern: sift-py-dist-py*-${{ matrix.platform.platform_tag }}
           path: platform_archives
           merge-multiple: false
     
@@ -188,7 +188,7 @@ jobs:
           mkdir -p merged/dist/deps
           
           # Extract and merge all archives for this platform
-          for zip in platform_archives/*/sift-py-dist-py*-${{ matrix.platform }}.zip; do
+          for zip in platform_archives/*/sift-py-dist-py*-${{ matrix.platform.platform_tag }}.zip; do
             echo "Processing archive: $zip"
             
             # Create fresh temp directory
@@ -212,11 +212,11 @@ jobs:
           
           # Create merged archive
           cd merged
-          zip -r "../sift-py-dist-py3-${{ matrix.platform }}.zip" dist/
+          zip -r "../sift-py-dist-py3-${{ matrix.platform.platform_tag }}.zip" dist/
 
       - name: Upload merged archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-py3-${{ matrix.platform }}
+          name: sift-py-dist-py3-${{ matrix.platform.platform_tag }}
           path: sift-py-dist-py3-*.zip
           retention-days: 7

From 32696133c56f08b4c4056657c9597cb15b7d072e Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 15:55:53 -0800
Subject: [PATCH 51/75] ok lets do it

---
 .github/workflows/python_build.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 0322243e9..d2b7df159 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -163,11 +163,12 @@ jobs:
           retention-days: 7
 
   merge_platform_archives:
-    name: Merge archives for ${{ matrix.platform.os }} (${{ matrix.platform.arch }})
+    name: Merge archives for ${{ matrix.os }} (${{ matrix.arch }})
     needs: [build_and_verify, get-matrix-config]
     runs-on: ubuntu-latest
     strategy:
-      matrix: ${{ fromJson(needs.get-matrix-config.outputs.platforms) }}
+      matrix:
+        platform: ${{ fromJson(needs.get-matrix-config.outputs.platforms) }}
     steps:
       - name: Download platform archives
         uses: actions/download-artifact@v4

From 859a8d17338f8465c45ebd5525446749897c3a97 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 17:06:38 -0800
Subject: [PATCH 52/75] some clean up

---
 .github/workflows/python_build.yaml | 34 ++++++-----------------------
 1 file changed, 7 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d2b7df159..9aff3e04c 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -28,25 +28,13 @@ jobs:
         with:
           script: |
             const matrix = [
-              {os: 'ubuntu', arch: 'x86_64', runner: 'ubuntu-latest', platform_tag: 'ubuntu-x86_64'},
-              {os: 'ubuntu', arch: 'aarch64', runner: 'ubuntu-latest', platform_tag: 'ubuntu-aarch64'},
-              {os: 'macos', arch: 'x86_64', runner: 'macos-latest', platform_tag: 'macos-x86_64'},
-              {os: 'macos', arch: 'arm64', runner: 'macos-14', platform_tag: 'macos-arm64'},
+              {os: 'ubuntu', arch: 'x86_64', runner: 'ubuntu-latest', platform_tag: 'linux_x86_64'},
+              {os: 'ubuntu', arch: 'aarch64', runner: 'ubuntu-latest', platform_tag: 'linux_aarch64'},
+              {os: 'macos', arch: 'x86_64', runner: 'macos-latest', platform_tag: 'macos_x86_64'},
+              {os: 'macos', arch: 'arm64', runner: 'macos-latest', platform_tag: 'macos_arm64'},
               {os: 'windows', arch: 'amd64', runner: 'windows-latest', platform_tag: 'win_amd64'}
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
-      - name: debug 
-        if: always()
-        run: |
-          echo "${{ steps.set-matrix.outputs.matrix }}"
-      - name: debug 2
-        if: always()
-        run: |
-          echo "${{ fromJson(steps.set-matrix.outputs.matrix)[0].runner }}" 
-      - name: debug 3
-        if: always()
-        run: |  
-          echo "${{ needs.get-matrix-config.outputs.matrix[0].runner }}" 
 
   build_and_verify:
     name: Build and verify on ${{ matrix.platform.os }} (${{ matrix.platform.arch }}) with Python ${{ matrix.python-version }}
@@ -74,10 +62,6 @@ jobs:
           python -m pip install --upgrade pip
           pip install build pip-tools twine
 
-      - name: debug 
-        run: |
-          python -c "import sysconfig;import re;print(re.sub(r'[-.]', '_', sysconfig.get_platform()))"
-
       - name: Generate requirements
         working-directory: python
         shell: bash
@@ -163,7 +147,7 @@ jobs:
           retention-days: 7
 
   merge_platform_archives:
-    name: Merge archives for ${{ matrix.os }} (${{ matrix.arch }})
+    name: Merge archives for ${{ matrix.platform.os }} (${{ matrix.platform.arch }})
     needs: [build_and_verify, get-matrix-config]
     runs-on: ubuntu-latest
     strategy:
@@ -177,11 +161,6 @@ jobs:
           path: platform_archives
           merge-multiple: false
     
-      - name: debug 
-        run: |
-          echo "{{ needs.build_and_verify.outputs.matrix-config }}"
-          python -c "import sysconfig;import re;print(re.sub(r'[-.]', '_', sysconfig.get_platform()))"
-
       - name: Merge archives
         shell: bash
         run: |
@@ -213,6 +192,7 @@ jobs:
           
           # Create merged archive
           cd merged
+          rm -rf deps/ # this is now an empty directory
           zip -r "../sift-py-dist-py3-${{ matrix.platform.platform_tag }}.zip" dist/
 
       - name: Upload merged archive
@@ -220,4 +200,4 @@ jobs:
         with:
           name: sift-py-dist-py3-${{ matrix.platform.platform_tag }}
           path: sift-py-dist-py3-*.zip
-          retention-days: 7
+          retention-days: 14

From baf91064449747f31cc6b6e9e7a7d3e6af80f7b1 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 22:47:42 -0800
Subject: [PATCH 53/75] some more cleanup

---
 .github/workflows/python_build.yaml | 71 +++++++++++++----------------
 1 file changed, 32 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 9aff3e04c..af387af6d 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -9,17 +9,24 @@ jobs:
     name: Get matrix configuration
     runs-on: ubuntu-latest
     outputs:
-      python-versions: ${{ steps.get-versions.outputs.versions }}
+      supported_python_versions: ${{ steps.get-python-versions.outputs.versions }}
       platforms: ${{ steps.set-matrix.outputs.matrix }}
+      sift_package_version: ${{ steps.get-sift-version.outputs.version }}
     steps:
       - uses: actions/checkout@v4
       
-      - name: Get Python versions from pyproject.toml
-        id: get-versions
+      - name: Get supported Python versions from pyproject.toml
+        id: get-python-versions
         run: |
           versions=$(grep "Programming Language :: Python :: " python/pyproject.toml | sed 's/.*Python :: \([0-9.]*\).*/\1/' | jq -R -s -c 'split("\n")[:-1]')
           echo "versions=$versions" >> $GITHUB_OUTPUT
 
+      - name: Get sift-stack-py package version from pyproject.toml
+        id: get-sift-version
+        run: |
+          version=$(grep '^version = ' python/pyproject.toml | sed 's/version = "\(.*\)"/\1/')
+          echo "version=$version" >> $GITHUB_OUTPUT
+
       # We define the platforms here so we can reuse the matrix in multiple jobs
       # This is a workaround for the fact that we can't use the same matrix in multiple jobs
       - name: Set platform matrix
@@ -75,15 +82,15 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          mkdir -p dist/deps
+          # Download all dependencies directly to dist directory
           # First download build dependencies
-          pip download -d dist/deps \
+          pip download -d dist \
             setuptools wheel Cython \
             --prefer-binary \
             --no-deps
 
           # Then download package dependencies
-          pip download -r requirements-all.txt -d dist/deps \
+          pip download -r requirements-all.txt -d dist \
             --python-version ${{ matrix.python-version }} \
             --prefer-binary \
             --no-deps
@@ -98,14 +105,14 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          twine check dist/*.whl
+          twine check dist/sift-stack-py*.whl
 
       - name: Test installations
         working-directory: python
         shell: bash
         run: |
           # Get the wheel file name
-          WHEEL_FILE=$(ls dist/*.whl)
+          WHEEL_FILE=$(ls dist/sift-stack-py*.whl)
           
           # Test base installation
           python -m venv test_venv_base
@@ -114,7 +121,7 @@ jobs:
           else
             source test_venv_base/bin/activate
           fi
-          pip install --no-index --find-links=dist/deps "$WHEEL_FILE"
+          pip install --no-index --find-links=dist/ "$WHEEL_FILE"
           deactivate
           rm -rf test_venv_base
 
@@ -125,7 +132,7 @@ jobs:
           else
             source test_venv_openssl/bin/activate
           fi
-          pip install --no-index --find-links=dist/deps "$WHEEL_FILE[openssl]"
+          pip install --no-index --find-links=dist/ "$WHEEL_FILE[openssl]"
           deactivate
           rm -rf test_venv_openssl
 
@@ -134,17 +141,17 @@ jobs:
         shell: bash
         run: |
           if [ "${{ matrix.platform.os }}" = "windows" ]; then
-            pwsh -Command "Compress-Archive -Path dist/* -DestinationPath dist/sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}.zip -Force"
+            pwsh -Command "Compress-Archive -Path dist/* -DestinationPath dist/sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}.zip -Force"
           else
-            cd dist && zip -r "sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}.zip" *
+            cd dist && zip -r "sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}.zip" *
           fi
 
       - name: Upload distribution archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}
+          name: sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py${{ matrix.python-version }}-${{ matrix.platform.platform_tag }}
           path: python/dist/sift-py-dist-*.zip
-          retention-days: 7
+          retention-days: 14
 
   merge_platform_archives:
     name: Merge archives for ${{ matrix.platform.os }} (${{ matrix.platform.arch }})
@@ -157,47 +164,33 @@ jobs:
       - name: Download platform archives
         uses: actions/download-artifact@v4
         with:
-          pattern: sift-py-dist-py*-${{ matrix.platform.platform_tag }}
+          pattern: sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py*-${{ matrix.platform.platform_tag }}
           path: platform_archives
           merge-multiple: false
     
       - name: Merge archives
         shell: bash
         run: |
-          # Create directory for merged wheels
-          mkdir -p merged/dist/deps
+          # Create directory for merged files
+          mkdir -p merged
           
           # Extract and merge all archives for this platform
-          for zip in platform_archives/*/sift-py-dist-py*-${{ matrix.platform.platform_tag }}.zip; do
+          for zip in platform_archives/*/sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py*-${{ matrix.platform.platform_tag }}.zip; do
             echo "Processing archive: $zip"
             
-            # Create fresh temp directory
-            rm -rf temp_extract
-            mkdir -p temp_extract
-            
-            # Extract and show contents
-            unzip -o "$zip" -d "temp_extract"
-            echo "Contents of temp_extract:"
-            ls -R temp_extract
-            
-            # Find all files recursively and copy them
-            find temp_extract -type f -exec cp {} merged/dist/ \;
-            
-            # Cleanup
-            rm -rf temp_extract
+            # Extract directly to merged directory
+            unzip -o "$zip" -d "merged"
+            echo "Contents after extracting $zip:"
+            ls -R merged
           done
           
-          echo "Contents of merged directory:"
-          ls -R merged/dist/
-          
           # Create merged archive
           cd merged
-          rm -rf deps/ # this is now an empty directory
-          zip -r "../sift-py-dist-py3-${{ matrix.platform.platform_tag }}.zip" dist/
+          zip -r "../sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py3-${{ matrix.platform.platform_tag }}.zip" *
 
       - name: Upload merged archive
         uses: actions/upload-artifact@v4
         with:
-          name: sift-py-dist-py3-${{ matrix.platform.platform_tag }}
-          path: sift-py-dist-py3-*.zip
+          name: sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py3-${{ matrix.platform.platform_tag }}
+          path: sift-py-dist-*.zip
           retention-days: 14

From c3b0ff3dda57017da0f8ca1134042842f0fd0219 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 22:50:03 -0800
Subject: [PATCH 54/75] fixzup

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index af387af6d..d05656a87 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -51,7 +51,7 @@ jobs:
       fail-fast: false
       matrix:
         platform: ${{ fromJson(needs.get-matrix-config.outputs.platforms) }}
-        python-version: ${{fromJson(needs.get-matrix-config.outputs.python-versions)}}
+        python-version: ${{fromJson(needs.get-matrix-config.outputs.supported_python_versions)}}
     outputs:
       matrix-config: ${{ toJson(matrix) }}
 

From c9c24f1ffc3fe8a512f8732dd109e674f91a8e68 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 22:52:52 -0800
Subject: [PATCH 55/75] maybe just maybe

---
 .github/workflows/python_build.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d05656a87..3317e4021 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -105,14 +105,14 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          twine check dist/sift-stack-py*.whl
+          twine check dist/sift_stack_py*.whl
 
       - name: Test installations
         working-directory: python
         shell: bash
         run: |
           # Get the wheel file name
-          WHEEL_FILE=$(ls dist/sift-stack-py*.whl)
+          WHEEL_FILE=$(ls dist/sift_stack_py*.whl)
           
           # Test base installation
           python -m venv test_venv_base

From 3628b95e782ec6541be1136b9386d1fdc2718faa Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 23:02:27 -0800
Subject: [PATCH 56/75] lets build wheels for everytihg

---
 .github/workflows/python_build.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 3317e4021..305b395cd 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -78,19 +78,19 @@ jobs:
           # so we need to build it from source which was causing issues on windows
           pip-compile pyproject.toml --extra=openssl -o requirements-all.txt
 
-      - name: Download dependencies
+      - name: Build dependency wheels
         working-directory: python
         shell: bash
         run: |
-          # Download all dependencies directly to dist directory
-          # First download build dependencies
-          pip download -d dist \
+          # Build wheels for all dependencies directly to dist directory
+          # First build wheels for build dependencies
+          pip wheel -w dist \
             setuptools wheel Cython \
             --prefer-binary \
             --no-deps
 
-          # Then download package dependencies
-          pip download -r requirements-all.txt -d dist \
+          # Then build wheels for package dependencies
+          pip wheel -r requirements-all.txt -w dist \
             --python-version ${{ matrix.python-version }} \
             --prefer-binary \
             --no-deps

From 70efaa1307e3cf3660d265a25204a6d8d2eb3b3f Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 23:05:35 -0800
Subject: [PATCH 57/75] fix args

---
 .github/workflows/python_build.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 305b395cd..2fe806325 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -91,7 +91,6 @@ jobs:
 
           # Then build wheels for package dependencies
           pip wheel -r requirements-all.txt -w dist \
-            --python-version ${{ matrix.python-version }} \
             --prefer-binary \
             --no-deps
 

From cd50547f7c87e8480f09cd9ded9ae95bcd7d70ed Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Tue, 10 Dec 2024 23:10:21 -0800
Subject: [PATCH 58/75] Lets try all extras again

---
 .github/workflows/python_build.yaml | 55 ++++++++++++++++-------------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 2fe806325..b15344f9e 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -73,10 +73,8 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          # Generate requirements file with only openssl extras
-          # npTDMS doesn't have wheels for all platforms, 
-          # so we need to build it from source which was causing issues on windows
-          pip-compile pyproject.toml --extra=openssl -o requirements-all.txt
+          # Generate requirements file with all extras
+          pip-compile pyproject.toml --all-extras -o requirements-all.txt
 
       - name: Build dependency wheels
         working-directory: python
@@ -113,27 +111,34 @@ jobs:
           # Get the wheel file name
           WHEEL_FILE=$(ls dist/sift_stack_py*.whl)
           
-          # Test base installation
-          python -m venv test_venv_base
-          if [ "${{ matrix.platform.os }}" = "windows" ]; then
-            source test_venv_base/Scripts/activate
-          else
-            source test_venv_base/bin/activate
-          fi
-          pip install --no-index --find-links=dist/ "$WHEEL_FILE"
-          deactivate
-          rm -rf test_venv_base
-
-          # Test openssl installation
-          python -m venv test_venv_openssl
-          if [ "${{ matrix.platform.os }}" = "windows" ]; then
-            source test_venv_openssl/Scripts/activate
-          else
-            source test_venv_openssl/bin/activate
-          fi
-          pip install --no-index --find-links=dist/ "$WHEEL_FILE[openssl]"
-          deactivate
-          rm -rf test_venv_openssl
+          # Function to test installation with specific extras
+          test_install() {
+            local extras=$1
+            echo "Testing installation with extras: ${extras:-none}"
+            python -m venv "test_venv_${extras:-base}"
+            if [ "${{ matrix.platform.os }}" = "windows" ]; then
+              source "test_venv_${extras:-base}/Scripts/activate"
+            else
+              source "test_venv_${extras:-base}/bin/activate"
+            fi
+            if [ -z "$extras" ]; then
+              pip install --no-index --find-links=dist/ "$WHEEL_FILE"
+            else
+              pip install --no-index --find-links=dist/ "$WHEEL_FILE[$extras]"
+            fi
+            deactivate
+            rm -rf "test_venv_${extras:-base}"
+          }
+          
+          # Test each combination
+          test_install ""  # base installation
+          test_install "openssl"
+          test_install "build"
+          test_install "development"
+          test_install "openssl,build"
+          test_install "openssl,development"
+          test_install "build,development"
+          test_install "openssl,build,development"
 
       - name: Create distribution archive
         working-directory: python

From ea208b3e256f980b92f92ae71839d2f207137e7e Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 00:40:24 -0800
Subject: [PATCH 59/75] add build workflow and github release to
 python_release.yaml bump rc

---
 .github/workflows/python_release.yaml | 48 ++++++++++++++++++++++++++-
 python/pyproject.toml                 |  2 +-
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index 310738490..fe9e873ea 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -8,9 +8,14 @@ jobs:
     if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags')
     uses: ./.github/workflows/python_ci.yaml
 
+  build-offline-archives:
+    if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags')
+    needs: python-ci
+    uses: ./.github/workflows/python_build.yaml
+
   publish-to-pypi:
     name: Upload release to PyPI
-    needs: python-ci
+    needs: [python-ci, build-offline-archives]
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -44,3 +49,44 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: ./python/dist/
+
+  create-github-release:
+    name: Create GitHub Release
+    needs: [build-offline-archives]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Download all platform archives
+        uses: actions/download-artifact@v4
+        with:
+          pattern: sift-py-dist-*-py3-*
+          path: platform_archives
+          merge-multiple: false
+
+      - name: Create Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG_NAME: ${{ github.ref_name }}
+        run: |
+          # Create release notes
+          cat > release_notes.md << 'EOL'
+          See [CHANGELOG.md](https://github.com/sift-stack/sift/blob/main/python/CHANGELOG.md) for details.
+
+          Offline archives are available for download for multiple platforms. Offline archives include wheels for all dependencies including build extras, e.g. openssl, development, and build.
+
+          Use `pip install sift-stack-py --find-links={path/to/archive} --no-index` to install.
+          EOL
+
+          # Create the release
+          gh release create "$TAG_NAME" \
+            --title "sift-stack-py $TAG_NAME" \
+            --notes-file release_notes.md
+
+          # Upload each platform archive
+          for archive in platform_archives/*/sift-py-dist-*-py3-*.zip; do
+            echo "Uploading $archive to release $TAG_NAME"
+            gh release upload "$TAG_NAME" "$archive" \
+              --clobber
+          done
+          
diff --git a/python/pyproject.toml b/python/pyproject.toml
index e792e77cc..0ea0fca59 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "sift_stack_py"
-version = "0.3.0-rc.6"
+version = "0.3.0-rc.7"
 description = "Python client library for the Sift API"
 requires-python = ">=3.8"
 readme = {file = "README.md", content-type = "text/markdown"}

From c15200917b55a27ffeb81145eab48055864a4635 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 00:40:48 -0800
Subject: [PATCH 60/75] REVERT ME dont push to pypi

---
 .github/workflows/python_release.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index fe9e873ea..25f57d3f6 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -45,10 +45,10 @@ jobs:
         run: |
           python -m build
 
-      - name: Publish package distributions to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: ./python/dist/
+      # - name: Publish package distributions to PyPI
+      #   uses: pypa/gh-action-pypi-publish@release/v1
+      #   with:
+      #     packages-dir: ./python/dist/
 
   create-github-release:
     name: Create GitHub Release

From 2c395fbc6f33dc934cf46a56370ac5ff4f32b6a7 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 00:49:04 -0800
Subject: [PATCH 61/75] remove pr trigger and add workflow_call

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index b15344f9e..2e00f9825 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -2,7 +2,7 @@ name: Offline Installation Archive
 
 on:
   workflow_dispatch:
-  pull_request:
+  workflow_call:
 
 jobs:
   get-matrix-config:

From 4c16ff5db1340f46d06433f18d014888085345b9 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 01:04:26 -0800
Subject: [PATCH 62/75] add git checkout before trying to create release

---
 .github/workflows/python_release.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index 25f57d3f6..e93633402 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -57,6 +57,9 @@ jobs:
     permissions:
       contents: write
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Download all platform archives
         uses: actions/download-artifact@v4
         with:

From 54826785bd999182ab14f21a8c3f05cdfa8a0b4a Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 01:17:09 -0800
Subject: [PATCH 63/75] uncomment the publish to pypi

---
 .github/workflows/python_release.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index e93633402..8c861174e 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -45,10 +45,10 @@ jobs:
         run: |
           python -m build
 
-      # - name: Publish package distributions to PyPI
-      #   uses: pypa/gh-action-pypi-publish@release/v1
-      #   with:
-      #     packages-dir: ./python/dist/
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: ./python/dist/
 
   create-github-release:
     name: Create GitHub Release

From c6c4568883a276dad0dcfe14ebe90fa46dbd9006 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 01:29:55 -0800
Subject: [PATCH 64/75] put npTDMS back in dependencies

A separate pr should put it in its own extras group
---
 python/pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyproject.toml b/python/pyproject.toml
index 0ea0fca59..5fbf0d85b 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -22,6 +22,7 @@ keywords = [
 ]
 dependencies = [
   "grpcio~=1.13",
+  "npTDMS~=1.9",
   "PyYAML~=6.0",
   "pandas~=2.0",
   "protobuf>=4.0",
@@ -47,7 +48,6 @@ Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md"
 [project.optional-dependencies]
 development = [
   "grpcio-testing~=1.13",
-  "npTDMS~=1.9",
   "mypy==1.10.0",
   "pyright==1.1.386",
   "pytest==8.2.2",

From cdc447b9f4c1178a692986b822cc795390f9b30f Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 09:34:58 -0800
Subject: [PATCH 65/75] add a new build_utils.py script to simplify the yaml
 logic

---
 .github/workflows/python_build.yaml   | 104 ++++++++++++----------
 .github/workflows/python_release.yaml |  45 +++++-----
 python/scripts/build_utils.py         | 123 ++++++++++++++++++++++++++
 3 files changed, 201 insertions(+), 71 deletions(-)
 create mode 100644 python/scripts/build_utils.py

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 2e00f9825..2a37e1300 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -43,17 +43,61 @@ jobs:
             ];
             core.setOutput('matrix', JSON.stringify(matrix));
 
+  build_wheel:
+    name: Build sift-stack-py distributions
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"  # Use lowest supported version for maximum compatibility
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Build distributions
+        working-directory: python
+        run: |
+          python -m build
+
+      - name: Verify distributions
+        working-directory: python
+        shell: bash
+        run: |
+          # Check all distributions with twine
+          twine check dist/*
+
+          # Verify we have a universal wheel
+          # We want to ensure that the wheel is compatible with all Python versions
+          # and all architectures. If this fails, we will need to update our build strategy to
+          # build separate wheels for each Python version and architecture.
+          WHEEL_NAME=$(ls dist/sift_stack_py*.whl)
+          if [[ ! $WHEEL_NAME =~ "py3-none-any.whl" ]]; then
+            echo "Error: Expected a universal wheel (py3-none-any) but got: $WHEEL_NAME"
+            exit 1
+          fi
+          echo "Verified universal wheel: $WHEEL_NAME"
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: sift-stack-py-dist
+          path: python/dist/*
+          retention-days: 14
+
   build_and_verify:
-    name: Build and verify on ${{ matrix.platform.os }} (${{ matrix.platform.arch }}) with Python ${{ matrix.python-version }}
-    needs: get-matrix-config
+    name: Build offline archive for ${{ matrix.platform.os }} (${{ matrix.platform.arch }}) with Python ${{ matrix.python-version }}
+    needs: [get-matrix-config, build_wheel]
     runs-on: ${{ matrix.platform.runner }}
     strategy:
       fail-fast: false
       matrix:
         platform: ${{ fromJson(needs.get-matrix-config.outputs.platforms) }}
         python-version: ${{fromJson(needs.get-matrix-config.outputs.supported_python_versions)}}
-    outputs:
-      matrix-config: ${{ toJson(matrix) }}
 
     steps:
       - uses: actions/checkout@v4
@@ -92,53 +136,21 @@ jobs:
             --prefer-binary \
             --no-deps
 
-      - name: Build sift-py wheel
-        working-directory: python
-        shell: bash
-        run: |
-          python -m build --wheel
-
-      - name: Verify wheel
-        working-directory: python
-        shell: bash
-        run: |
-          twine check dist/sift_stack_py*.whl
+      - name: Download sift-stack-py wheel
+        uses: actions/download-artifact@v4
+        with:
+          name: sift-stack-py-wheel
+          path: python/dist/
 
       - name: Test installations
         working-directory: python
         shell: bash
         run: |
-          # Get the wheel file name
-          WHEEL_FILE=$(ls dist/sift_stack_py*.whl)
-          
-          # Function to test installation with specific extras
-          test_install() {
-            local extras=$1
-            echo "Testing installation with extras: ${extras:-none}"
-            python -m venv "test_venv_${extras:-base}"
-            if [ "${{ matrix.platform.os }}" = "windows" ]; then
-              source "test_venv_${extras:-base}/Scripts/activate"
-            else
-              source "test_venv_${extras:-base}/bin/activate"
-            fi
-            if [ -z "$extras" ]; then
-              pip install --no-index --find-links=dist/ "$WHEEL_FILE"
-            else
-              pip install --no-index --find-links=dist/ "$WHEEL_FILE[$extras]"
-            fi
-            deactivate
-            rm -rf "test_venv_${extras:-base}"
-          }
-          
-          # Test each combination
-          test_install ""  # base installation
-          test_install "openssl"
-          test_install "build"
-          test_install "development"
-          test_install "openssl,build"
-          test_install "openssl,development"
-          test_install "build,development"
-          test_install "openssl,build,development"
+          pip install wheel
+          python scripts/build_utils.py \
+            --dist-dir dist \
+            --package-name sift-stack-py \
+            --is-windows ${{ matrix.platform.os == 'windows' }}
 
       - name: Create distribution archive
         working-directory: python
diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index 8c861174e..d0c90152f 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -17,38 +17,22 @@ jobs:
     name: Upload release to PyPI
     needs: [python-ci, build-offline-archives]
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: python
     environment:
       name: pypi
       url: https://pypi.org/p/sift_py
     permissions:
       id-token: write  
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - name: Download distributions
+        uses: actions/download-artifact@v4
         with:
-          python-version: "3.8"
-
-      - name: Pip install
-        run: |
-          python -m pip install --upgrade pip
-          pip install '.[build]'
-          pip install .
-
-      - name: Build distributions
-        working-directory: python
-        run: |
-          python -m build
+          name: sift-stack-py-dist
+          path: python/dist/
 
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          packages-dir: ./python/dist/
+          packages-dir: python/dist/
 
   create-github-release:
     name: Create GitHub Release
@@ -60,6 +44,12 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: sift-stack-py-dist
+          path: python/dist/
+
       - name: Download all platform archives
         uses: actions/download-artifact@v4
         with:
@@ -86,10 +76,15 @@ jobs:
             --title "sift-stack-py $TAG_NAME" \
             --notes-file release_notes.md
 
-          # Upload each platform archive
+          # Upload Python package distributions
+          for dist in python/dist/*; do
+            echo "Uploading distribution: $dist"
+            gh release upload "$TAG_NAME" "$dist" --clobber
+          done
+
+          # Upload platform archives
           for archive in platform_archives/*/sift-py-dist-*-py3-*.zip; do
-            echo "Uploading $archive to release $TAG_NAME"
-            gh release upload "$TAG_NAME" "$archive" \
-              --clobber
+            echo "Uploading archive: $archive"
+            gh release upload "$TAG_NAME" "$archive" --clobber
           done
           
diff --git a/python/scripts/build_utils.py b/python/scripts/build_utils.py
new file mode 100644
index 000000000..5d54c6aba
--- /dev/null
+++ b/python/scripts/build_utils.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import venv
+from itertools import combinations
+from pathlib import Path
+from typing import List, Optional
+from wheel.pkginfo import read_pkg_info_bytes
+from zipfile import ZipFile
+
+
+def get_extras_from_wheel(wheel_path: str) -> List[str]:
+    """Extract the list of extras from a wheel's metadata.
+
+    Args:
+        wheel_path: Path to the wheel file to inspect.
+
+    Returns:
+        List of extra names declared in the wheel metadata.
+    """
+    with ZipFile(wheel_path) as wheel:
+        for name in wheel.namelist():
+            if name.endswith('.dist-info/METADATA'):
+                metadata = read_pkg_info_bytes(wheel.read(name))
+                return [
+                    line.split(':')[1].strip()
+                    for line in metadata.get_all('Provides-Extra') or []
+                ]
+    return []
+
+
+def get_extra_combinations(extras: List[str]) -> List[str]:
+    """Generate all possible combinations of extras.
+
+    Args:
+        extras: List of extra names to generate combinations from.
+
+    Returns:
+        List of comma-separated strings representing each combination of extras.
+    """
+    all_combinations = []
+    for r in range(len(extras) + 1):
+        all_combinations.extend(','.join(c) for c in combinations(extras, r))
+    return all_combinations
+
+
+def test_install(
+    package_name: str,
+    extras: Optional[str],
+    dist_dir: str,
+    venv_dir: str,
+    is_windows: bool
+) -> None:
+    """Test package installation with given extras in a fresh venv.
+
+    Args:
+        package_name: Name of the package to install.
+        extras: Optional comma-separated string of extras to install.
+        dist_dir: Directory containing wheel and dependencies.
+        venv_dir: Directory to create virtual environment in.
+        is_windows: Whether running on Windows platform.
+    """
+    print(f"Testing installation with extras: {extras or 'none'}")
+    
+    # Create and activate venv
+    venv.create(venv_dir, with_pip=True)
+    
+    # Build activation command based on OS
+    if is_windows:
+        activate_script = os.path.join(venv_dir, "Scripts", "activate")
+    else:
+        activate_script = os.path.join(venv_dir, "bin", "activate")
+
+    # Build installation command
+    if extras:
+        install_cmd = f'pip install --no-index --find-links="{dist_dir}" "{package_name}[{extras}]"'
+    else:
+        install_cmd = f'pip install --no-index --find-links="{dist_dir}" {package_name}'
+
+    # Run installation in the venv
+    full_cmd = f'source "{activate_script}" && {install_cmd} && deactivate'
+    subprocess.run(full_cmd, shell=True, check=True, executable='/bin/bash')
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Test package installation with all extra combinations')
+    parser.add_argument('--dist-dir', required=True, help='Directory containing wheel and dependencies')
+    parser.add_argument('--package-name', required=True, help='Name of the package to install')
+    parser.add_argument('--is-windows', action='store_true', help='Whether running on Windows')
+    args = parser.parse_args()
+
+    dist_dir = Path(args.dist_dir)
+    wheel_file = next(dist_dir.glob(f"{args.package_name.replace('-', '_')}*.whl"))
+    
+    # Get all extras from the wheel
+    extras = get_extras_from_wheel(str(wheel_file))
+    combinations = get_extra_combinations(extras)
+    
+    # Test base installation first
+    test_install(
+        package_name=args.package_name,
+        extras=None,
+        dist_dir=str(dist_dir),
+        venv_dir='test_venv_base',
+        is_windows=args.is_windows
+    )
+
+    # Test each combination of extras
+    for combo in combinations:
+        if combo:  # Skip empty string from base combination
+            test_install(
+                package_name=args.package_name,
+                extras=combo,
+                dist_dir=str(dist_dir),
+                venv_dir=f'test_venv_{combo.replace(",", "_")}',
+                is_windows=args.is_windows
+            )
+
+
+if __name__ == '__main__':
+    main() 
\ No newline at end of file

From 6e98a5251ac6ff97358ef8ac10c8794da0bffd34 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 09:35:45 -0800
Subject: [PATCH 66/75] revert me

comment out pypi publish
---
 .github/workflows/python_release.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index d0c90152f..51ef99a17 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -29,10 +29,10 @@ jobs:
           name: sift-stack-py-dist
           path: python/dist/
 
-      - name: Publish package distributions to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: python/dist/
+      # - name: Publish package distributions to PyPI
+      #   uses: pypa/gh-action-pypi-publish@release/v1
+      #   with:
+      #     packages-dir: python/dist/
 
   create-github-release:
     name: Create GitHub Release

From f8ee5656fb1455ea2d24ccba23105a8f9385e9ef Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 09:44:14 -0800
Subject: [PATCH 67/75] lint and format

---
 python/scripts/build_utils.py | 48 +++++++++++++++++------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/python/scripts/build_utils.py b/python/scripts/build_utils.py
index 5d54c6aba..06d7fe830 100644
--- a/python/scripts/build_utils.py
+++ b/python/scripts/build_utils.py
@@ -7,9 +7,10 @@
 from itertools import combinations
 from pathlib import Path
 from typing import List, Optional
-from wheel.pkginfo import read_pkg_info_bytes
 from zipfile import ZipFile
 
+from wheel.pkginfo import read_pkg_info_bytes
+
 
 def get_extras_from_wheel(wheel_path: str) -> List[str]:
     """Extract the list of extras from a wheel's metadata.
@@ -22,11 +23,10 @@ def get_extras_from_wheel(wheel_path: str) -> List[str]:
     """
     with ZipFile(wheel_path) as wheel:
         for name in wheel.namelist():
-            if name.endswith('.dist-info/METADATA'):
+            if name.endswith(".dist-info/METADATA"):
                 metadata = read_pkg_info_bytes(wheel.read(name))
                 return [
-                    line.split(':')[1].strip()
-                    for line in metadata.get_all('Provides-Extra') or []
+                    line.split(":")[1].strip() for line in metadata.get_all("Provides-Extra") or []
                 ]
     return []
 
@@ -42,16 +42,12 @@ def get_extra_combinations(extras: List[str]) -> List[str]:
     """
     all_combinations = []
     for r in range(len(extras) + 1):
-        all_combinations.extend(','.join(c) for c in combinations(extras, r))
+        all_combinations.extend(",".join(c) for c in combinations(extras, r))
     return all_combinations
 
 
 def test_install(
-    package_name: str,
-    extras: Optional[str],
-    dist_dir: str,
-    venv_dir: str,
-    is_windows: bool
+    package_name: str, extras: Optional[str], dist_dir: str, venv_dir: str, is_windows: bool
 ) -> None:
     """Test package installation with given extras in a fresh venv.
 
@@ -63,10 +59,10 @@ def test_install(
         is_windows: Whether running on Windows platform.
     """
     print(f"Testing installation with extras: {extras or 'none'}")
-    
+
     # Create and activate venv
     venv.create(venv_dir, with_pip=True)
-    
+
     # Build activation command based on OS
     if is_windows:
         activate_script = os.path.join(venv_dir, "Scripts", "activate")
@@ -81,30 +77,34 @@ def test_install(
 
     # Run installation in the venv
     full_cmd = f'source "{activate_script}" && {install_cmd} && deactivate'
-    subprocess.run(full_cmd, shell=True, check=True, executable='/bin/bash')
+    subprocess.run(full_cmd, shell=True, check=True, executable="/bin/bash")
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Test package installation with all extra combinations')
-    parser.add_argument('--dist-dir', required=True, help='Directory containing wheel and dependencies')
-    parser.add_argument('--package-name', required=True, help='Name of the package to install')
-    parser.add_argument('--is-windows', action='store_true', help='Whether running on Windows')
+    parser = argparse.ArgumentParser(
+        description="Test package installation with all extra combinations"
+    )
+    parser.add_argument(
+        "--dist-dir", required=True, help="Directory containing wheel and dependencies"
+    )
+    parser.add_argument("--package-name", required=True, help="Name of the package to install")
+    parser.add_argument("--is-windows", action="store_true", help="Whether running on Windows")
     args = parser.parse_args()
 
     dist_dir = Path(args.dist_dir)
     wheel_file = next(dist_dir.glob(f"{args.package_name.replace('-', '_')}*.whl"))
-    
+
     # Get all extras from the wheel
     extras = get_extras_from_wheel(str(wheel_file))
     combinations = get_extra_combinations(extras)
-    
+
     # Test base installation first
     test_install(
         package_name=args.package_name,
         extras=None,
         dist_dir=str(dist_dir),
-        venv_dir='test_venv_base',
-        is_windows=args.is_windows
+        venv_dir="test_venv_base",
+        is_windows=args.is_windows,
     )
 
     # Test each combination of extras
@@ -115,9 +115,9 @@ def main():
                 extras=combo,
                 dist_dir=str(dist_dir),
                 venv_dir=f'test_venv_{combo.replace(",", "_")}',
-                is_windows=args.is_windows
+                is_windows=args.is_windows,
             )
 
 
-if __name__ == '__main__':
-    main() 
\ No newline at end of file
+if __name__ == "__main__":
+    main()

From a0f6414603676e6e1f2be03974ded9461bcc940a Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 10:08:33 -0800
Subject: [PATCH 68/75] fixup

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 2a37e1300..f024e1e72 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -139,7 +139,7 @@ jobs:
       - name: Download sift-stack-py wheel
         uses: actions/download-artifact@v4
         with:
-          name: sift-stack-py-wheel
+          name: sift-stack-py-dist
           path: python/dist/
 
       - name: Test installations

From e4c8da99825838f62bc4a72e0bfea86d642e9459 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 10:18:42 -0800
Subject: [PATCH 69/75] use std lib to parse wheel for supported extras

---
 .github/workflows/python_build.yaml |  1 -
 python/scripts/build_utils.py       | 22 +++++++++++++---------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index f024e1e72..c6397cc87 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -146,7 +146,6 @@ jobs:
         working-directory: python
         shell: bash
         run: |
-          pip install wheel
           python scripts/build_utils.py \
             --dist-dir dist \
             --package-name sift-stack-py \
diff --git a/python/scripts/build_utils.py b/python/scripts/build_utils.py
index 06d7fe830..c5fad90d2 100644
--- a/python/scripts/build_utils.py
+++ b/python/scripts/build_utils.py
@@ -9,8 +9,6 @@
 from typing import List, Optional
 from zipfile import ZipFile
 
-from wheel.pkginfo import read_pkg_info_bytes
-
 
 def get_extras_from_wheel(wheel_path: str) -> List[str]:
     """Extract the list of extras from a wheel's metadata.
@@ -22,13 +20,19 @@ def get_extras_from_wheel(wheel_path: str) -> List[str]:
         List of extra names declared in the wheel metadata.
     """
     with ZipFile(wheel_path) as wheel:
-        for name in wheel.namelist():
-            if name.endswith(".dist-info/METADATA"):
-                metadata = read_pkg_info_bytes(wheel.read(name))
-                return [
-                    line.split(":")[1].strip() for line in metadata.get_all("Provides-Extra") or []
-                ]
-    return []
+        # Find the METADATA file in the .dist-info directory
+        metadata_file = next(
+            name for name in wheel.namelist() 
+            if name.endswith('.dist-info/METADATA')
+        )
+        metadata = wheel.read(metadata_file).decode('utf-8')
+        
+        # Parse Provides-Extra lines from metadata
+        extras = []
+        for line in metadata.splitlines():
+            if line.startswith('Provides-Extra:'):
+                extras.append(line.split(':', 1)[1].strip())
+        return extras
 
 
 def get_extra_combinations(extras: List[str]) -> List[str]:

From fb40c1954f742845573a4c468d28394870802075 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 10:25:20 -0800
Subject: [PATCH 70/75] fixup

---
 python/scripts/build_utils.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/python/scripts/build_utils.py b/python/scripts/build_utils.py
index c5fad90d2..646882a67 100644
--- a/python/scripts/build_utils.py
+++ b/python/scripts/build_utils.py
@@ -22,16 +22,15 @@ def get_extras_from_wheel(wheel_path: str) -> List[str]:
     with ZipFile(wheel_path) as wheel:
         # Find the METADATA file in the .dist-info directory
         metadata_file = next(
-            name for name in wheel.namelist() 
-            if name.endswith('.dist-info/METADATA')
+            name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")
         )
-        metadata = wheel.read(metadata_file).decode('utf-8')
-        
+        metadata = wheel.read(metadata_file).decode("utf-8")
+
         # Parse Provides-Extra lines from metadata
         extras = []
         for line in metadata.splitlines():
-            if line.startswith('Provides-Extra:'):
-                extras.append(line.split(':', 1)[1].strip())
+            if line.startswith("Provides-Extra:"):
+                extras.append(line.split(":", 1)[1].strip())
         return extras
 
 

From 52067c7db1914dbe77ba6ae7577e3551087b7d80 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 10:29:59 -0800
Subject: [PATCH 71/75] fix flag

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index c6397cc87..176f0e5da 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -149,7 +149,7 @@ jobs:
           python scripts/build_utils.py \
             --dist-dir dist \
             --package-name sift-stack-py \
-            --is-windows ${{ matrix.platform.os == 'windows' }}
+            ${{ matrix.platform.os == 'windows' && '--is-windows' || '' }}
 
       - name: Create distribution archive
         working-directory: python

From c1f4b24d23ee3be6decb1331a154a1058cb4935c Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 10:56:55 -0800
Subject: [PATCH 72/75] fix windows

---
 .github/workflows/python_build.yaml |  2 +-
 python/scripts/build_utils.py       | 19 ++++++++++---------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index 176f0e5da..e9a2f5397 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -111,7 +111,7 @@ jobs:
         shell: bash
         run: |
           python -m pip install --upgrade pip
-          pip install build pip-tools twine
+          pip install build pip-tools
 
       - name: Generate requirements
         working-directory: python
diff --git a/python/scripts/build_utils.py b/python/scripts/build_utils.py
index 646882a67..b519d6141 100644
--- a/python/scripts/build_utils.py
+++ b/python/scripts/build_utils.py
@@ -66,21 +66,22 @@ def test_install(
     # Create and activate venv
     venv.create(venv_dir, with_pip=True)
 
-    # Build activation command based on OS
-    if is_windows:
-        activate_script = os.path.join(venv_dir, "Scripts", "activate")
-    else:
-        activate_script = os.path.join(venv_dir, "bin", "activate")
-
     # Build installation command
     if extras:
         install_cmd = f'pip install --no-index --find-links="{dist_dir}" "{package_name}[{extras}]"'
     else:
         install_cmd = f'pip install --no-index --find-links="{dist_dir}" {package_name}'
 
-    # Run installation in the venv
-    full_cmd = f'source "{activate_script}" && {install_cmd} && deactivate'
-    subprocess.run(full_cmd, shell=True, check=True, executable="/bin/bash")
+    if is_windows:
+        # Windows uses different activation and command syntax
+        activate_script = os.path.join(venv_dir, "Scripts", "activate.bat")
+        full_cmd = f'"{activate_script}" && {install_cmd} && deactivate'
+        subprocess.run(full_cmd, shell=True, check=True)
+    else:
+        # Unix systems use bash
+        activate_script = os.path.join(venv_dir, "bin", "activate")
+        full_cmd = f'source "{activate_script}" && {install_cmd} && deactivate'
+        subprocess.run(full_cmd, shell=True, check=True, executable="/bin/bash")
 
 
 def main():

From a4f6b02b09bc7d87472a56e602791974d87bd145 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 11:01:31 -0800
Subject: [PATCH 73/75] pr feedback

---
 .github/workflows/python_build.yaml   | 35 ++++++++++++++++++---------
 .github/workflows/python_release.yaml |  4 +--
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index e9a2f5397..d032cb96a 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -5,6 +5,12 @@ on:
   workflow_call:
 
 jobs:
+  # Run ci on workflow_dispatch to ensure we have a clean build.
+  # If using workflow_call, we expect the caller (python_release.yaml) to have already run ci.
+  python-ci:
+    if: github.event_name == 'workflow_dispatch'
+    uses: ./.github/workflows/python_ci.yaml
+
   get-matrix-config:
     name: Get matrix configuration
     runs-on: ubuntu-latest
@@ -14,7 +20,7 @@ jobs:
       sift_package_version: ${{ steps.get-sift-version.outputs.version }}
     steps:
       - uses: actions/checkout@v4
-      
+
       - name: Get supported Python versions from pyproject.toml
         id: get-python-versions
         run: |
@@ -52,7 +58,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"  # Use lowest supported version for maximum compatibility
+          python-version: "3.8" # Use lowest supported version for maximum compatibility
 
       - name: Install build tools
         run: |
@@ -182,30 +188,37 @@ jobs:
           pattern: sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py*-${{ matrix.platform.platform_tag }}
           path: platform_archives
           merge-multiple: false
-    
+
       - name: Merge archives
         shell: bash
         run: |
           # Create directory for merged files
           mkdir -p merged
-          
+
           # Extract and merge all archives for this platform
           for zip in platform_archives/*/sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py*-${{ matrix.platform.platform_tag }}.zip; do
             echo "Processing archive: $zip"
-            
-            # Extract directly to merged directory
             unzip -o "$zip" -d "merged"
             echo "Contents after extracting $zip:"
             ls -R merged
           done
-          
-          # Create merged archive
+
+          # Create base name for archives
+          ARCHIVE_BASE="sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py3-${{ matrix.platform.platform_tag }}"
+
+          # Create zip archive
           cd merged
-          zip -r "../sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py3-${{ matrix.platform.platform_tag }}.zip" *
+          zip -r "../${ARCHIVE_BASE}.zip" *
+
+          # Create tar.gz archive
+          tar -czf "../${ARCHIVE_BASE}.tar.gz" *
+          cd ..
 
-      - name: Upload merged archive
+      - name: Upload merged archives
         uses: actions/upload-artifact@v4
         with:
           name: sift-py-dist-${{ needs.get-matrix-config.outputs.sift_package_version }}-py3-${{ matrix.platform.platform_tag }}
-          path: sift-py-dist-*.zip
+          path: |
+            sift-py-dist-*.zip
+            sift-py-dist-*.tar.gz
           retention-days: 14
diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index 51ef99a17..152921eb8 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -82,8 +82,8 @@ jobs:
             gh release upload "$TAG_NAME" "$dist" --clobber
           done
 
-          # Upload platform archives
-          for archive in platform_archives/*/sift-py-dist-*-py3-*.zip; do
+          # Upload platform archives (both .zip and .tar.gz)
+          for archive in platform_archives/*/sift-py-dist-*-py3-*.{zip,tar.gz}; do
             echo "Uploading archive: $archive"
             gh release upload "$TAG_NAME" "$archive" --clobber
           done

From 5884f0ee1d746d3560324a14400c44baebd4b6b8 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 11:21:10 -0800
Subject: [PATCH 74/75] dont run ci on release

---
 .github/workflows/python_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python_build.yaml b/.github/workflows/python_build.yaml
index d032cb96a..81b9177b6 100644
--- a/.github/workflows/python_build.yaml
+++ b/.github/workflows/python_build.yaml
@@ -8,7 +8,7 @@ jobs:
   # Run ci on workflow_dispatch to ensure we have a clean build.
   # If using workflow_call, we expect the caller (python_release.yaml) to have already run ci.
   python-ci:
-    if: github.event_name == 'workflow_dispatch'
+    if: github.event_name == 'workflow_dispatch' && !github.event.workflow
     uses: ./.github/workflows/python_ci.yaml
 
   get-matrix-config:

From c775584fb4f03b99f9231a5ca2403ace7fdd6544 Mon Sep 17 00:00:00 2001
From: Kevin Limburg <kevin@siftstack.com>
Date: Wed, 11 Dec 2024 11:37:42 -0800
Subject: [PATCH 75/75] final fix up

---
 .github/workflows/python_release.yaml | 8 ++++----
 python/pyproject.toml                 | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/python_release.yaml b/.github/workflows/python_release.yaml
index 152921eb8..3c8be0286 100644
--- a/.github/workflows/python_release.yaml
+++ b/.github/workflows/python_release.yaml
@@ -29,10 +29,10 @@ jobs:
           name: sift-stack-py-dist
           path: python/dist/
 
-      # - name: Publish package distributions to PyPI
-      #   uses: pypa/gh-action-pypi-publish@release/v1
-      #   with:
-      #     packages-dir: python/dist/
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: python/dist/
 
   create-github-release:
     name: Create GitHub Release
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 5fbf0d85b..a86b4bff0 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "sift_stack_py"
-version = "0.3.0-rc.7"
+version = "0.3.0-rc.6"
 description = "Python client library for the Sift API"
 requires-python = ">=3.8"
 readme = {file = "README.md", content-type = "text/markdown"}