Merge pull request #37 from SpikeInterface/auto-build-cambridge-library

Unify scripts to check probes + autogen CN library

Author: samuelgarcia

.github/workflows/check_for_changes_in_NP_jsons.yml

@@ -38,19 +38,16 @@ jobs:
           pip install ./probeinterface
 
 
-      - name: Generate full NP library
+      - name: Generate full NP library and check for changes
         run: |
           cd scripts/
+          # generate full library
           python ../probeinterface/resources/generate_neuropixels_library.py
-
-      # Check for any new probes
-      - name: Run local script
-        run: |
-          cd scripts/
-          python check_for_json_changes_in_NP_probes.py
+          # check for json changes (except probeinterface version)
+          python check_for_json_changes_in_probes.py ../imec/ ./neuropixels_library_generated/
+          # clean up
           rm -r neuropixels_library_generated/
           cd ..
-
           rm -r ./probeinterface
 
       - name: Commit changes if any

…ithub/workflows/check_for_new_probes.yml …ub/workflows/check_for_new_NP_probes.yml.github/workflows/check_for_new_probes.yml renamed to .github/workflows/check_for_new_NP_probes.yml .github/workflows/check_for_new_probes.yml renamed to .github/workflows/check_for_new_NP_probes.yml

@@ -0,0 +1,75 @@
+name: Generate Cambridge Neurotech library
+
+on:
+  schedule:
+    - cron: '0 1 * * 1' # Every Monday at 01:00 UTC
+  workflow_dispatch:
+
+# Grant permissions for the job to write to contents (push) and create PRs.
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  build-and-pr:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out local repository
+        uses: actions/checkout@v4
+
+      # Set up a Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10' 
+
+      #  Clone dev version of probeinterface
+      - name: Clone external repositories
+        run: |
+          # probeinterface
+          git clone https://github.com/spikeinterface/probeinterface ./probeinterface
+
+          # cambridge neurotech probe maps
+          git clone https://github.com/cambridge-neurotech/probe_maps.git ./probe_maps
+
+      - name: Install probeinterface and matplotlib
+        run: pip install ./probeinterface matplotlib
+
+      - name: Generate full cambridge library and check for changes
+        run: |
+          cd scripts/
+          python ../probeinterface/resources/generate_cambridgeneurotech_library.py ../probe_maps/ ./cambridge_library_generated/
+
+          # check for json changes (except probeinterface version)
+          python check_for_json_changes_in_probes.py ../cambridgeneurotech/ ./cambridge_library_generated/
+          # clean up
+          rm -r cambridge_library_generated/
+          cd ..
+          rm -r ./probeinterface
+
+      - name: Commit changes if any
+        id: commit
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+
+          git add cambridgeneurotech/*
+
+          # Only commit if there are changes
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+            echo "changes=false" >> $GITHUB_OUTPUT
+          else
+            git commit -m "Update json files for NP probes"
+            echo "changes=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create pull request to add probes
+        if: steps.commit.outputs.changes == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          title: "Update Cambridge Neurotech json files"
+          body: "This PR updates the Neuropixel probes in probeinterace library, based on new data from the ProbeTable repository or because of a new ProbeInterface release."
+          branch-suffix: short-commit-hash
+          base: "main"

.github/workflows/generate_cambridge_neurotech_library.yml

@@ -0,0 +1,44 @@
+from argparse import ArgumentParser
+from pathlib import Path
+import shutil
+
+
+parser = ArgumentParser(description="Check for JSON changes in probes from manufacturer")
+
+parser.add_argument(
+    "old_dir",
+    type=str,
+    help="Path to the old probes directory",
+)
+
+parser.add_argument(
+    "new_dir",
+    type=str,
+    help="Path to the new probes directory",
+)
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    old_dir = Path(args.old_dir)
+    new_dir = Path(args.new_dir)
+    for temp_probe_directory in new_dir.iterdir():
+        probe_name = str(temp_probe_directory.name)
+
+        temp_probe_json_path = temp_probe_directory / (probe_name + '.json')
+        old_probe_json_path = old_dir / probe_name / (probe_name + '.json')
+
+        if old_probe_json_path.is_file():
+            with open(temp_probe_json_path, 'r') as f1, open(old_probe_json_path, 'r') as f2:
+                # Read in json files
+                lines1 = f1.readlines()
+                lines2 = f2.readlines()
+
+            # We don't want to update the probes just because of a probeinterface version update.
+            # The probeinterface version is stored on the 3rd line of the json file, so we only
+            # compare the json files from line 3 and down.
+            if lines1[3:] == lines2[3:]:
+                continue
+            else:
+                shutil.copy(f"{temp_probe_json_path}", f"../imec/{probe_name}")
+