Sync

.claude/settings.local.json

@@ -21,7 +21,11 @@
       "Bash(rm:*)",
       "Bash(curl:*)",
       "Bash(gh:*)",
-      "Bash(./git.sh:*)"
+      "Bash(./git.sh:*)",
+      "Bash(yarn build)",
+      "Bash(git rm:*)",
+      "Bash(git checkout:*)",
+      "Bash(awk:*)"
     ],
     "deny": [
       "Read(../**)"

.github/workflows/vector_sync.yml

@@ -1,18 +1,13 @@
 name: VectorDB Sync
 
 on:
-  # Run when a PR is merged into main
   pull_request:
     types: [closed]
-
-  # Run on direct pushes to main branch
   push:
-    branches:
-      - main
+    branches: [ main ]
 
 jobs:
   sync:
-    # For PR: only if merged into main
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') ||
       (github.event_name == 'push' && github.ref == 'refs/heads/main')
@@ -33,45 +28,38 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt || true
+          # Try repo-specific requirements
+          pip install -r codechat/requirements.txt || true
+          # Core sync deps (serverless Pinecone SDK)
+          pip install --upgrade tiktoken tree_sitter tqdm pyyaml pandas pinecone openai
 
-      - name: Detect changed files (root + submodules)
+      - name: Run VectorDB Sync (commit-range)
+        env:
+          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        shell: bash
         run: |
-          echo "🔎 Detecting changes..."
-          if [ "${{ github.event_name }}" == "push" ]; then
-            echo "Push to main branch detected"
-            git diff --name-status HEAD^ HEAD > changed_files.txt || true
-          else
-            echo "Pull request merged into main detected"
-            git diff --name-status origin/main HEAD > changed_files.txt || true
+          set -euo pipefail
+          # Skip if sync script not present (e.g., codechat PR not merged yet)
+          if [ ! -f codechat/vectordb_sync.py ]; then
+            echo "[skip] codechat/vectordb_sync.py not found; skipping VectorDB sync."
+            exit 0
           fi
 
-          cat changed_files.txt
-
-          echo "🔎 Detecting submodule pointer changes..."
-          git diff --submodule > submodule_changes.txt || true
-          cat submodule_changes.txt
-
-          echo "🔎 Expanding submodule changes into file-level diffs..."
-          while read -r line; do
-            submodule=$(echo "$line" | awk '{print $1}')
-            oldsha=$(echo "$line" | awk '{print $2}')
-            newsha=$(echo "$line" | awk '{print $3}')
-            if [ -n "$submodule" ] && [ -n "$oldsha" ] && [ -n "$newsha" ]; then
-              echo "📂 Submodule: $submodule ($oldsha → $newsha)"
-              pushd $submodule > /dev/null
-              git fetch origin $oldsha $newsha || true
-              git diff --name-status $oldsha $newsha | sed "s|^|$submodule/|" >> ../changed_files.txt
-              popd > /dev/null
-            fi
-          done < <(git diff --submodule | grep Submodule)
-
-          echo "✅ Final changed files list:"
-          cat changed_files.txt
-
-#      - name: Run VectorDB Sync
-#        env:
-#          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
-#          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-#        run: |
-#          python scripts/sync_pinecone.py changed_files.txt
+          # Skip if required secrets not provided
+          if [ -z "${PINECONE_API_KEY:-}" ] || [ -z "${OPENAI_API_KEY:-}" ]; then
+            echo "[skip] Missing PINECONE_API_KEY or OPENAI_API_KEY; skipping VectorDB sync."
+            exit 0
+          fi
+          if [ "${{ github.event_name }}" = "push" ]; then
+            BASE="${{ github.event.before }}"; HEAD="${{ github.sha }}"
+          else
+            BASE="${{ github.event.pull_request.base.sha }}"
+            HEAD="${{ github.event.pull_request.merge_commit_sha }}"
+            if [ -z "$HEAD" ]; then HEAD="${{ github.sha }}"; fi
+          fi
+          if [ "$BASE" = "0000000000000000000000000000000000000000" ] || [ -z "$BASE" ]; then
+            BASE="${HEAD}^"
+          fi
+          echo "[info] Sync range: $BASE..$HEAD"
+          python codechat/vectordb_sync.py --from-commit "$BASE" --to-commit "$HEAD" --repo-root .

.gitignore

@@ -1,17 +1,27 @@
-# Repos that are not submodules
-profile
-exiobase
-trade-data
-useeio.js
-io
-codechat
-membersense
-data-commons
-data-pipeline
+# Extra repos that are not submodules
+apps
+community
 community-data
 community-timelines
+data-commons
+data-pipeline
+display
+explore
+membersense
+nisar
+panels
+products-big-files-to-delete
+reality.streamlit*
+requests
+resources
+trade-data
+topojson
+useeio
+useeio-json
+useeio.js
+wiki
 
-# Configuration files with sensitive data
+# Configuration files
 config/settings.js
 config/keys.json
 config/.env
@@ -34,13 +44,14 @@ pids
 *.pid
 *.seed
 *.pid.lock
+__pycache__
 
-# Coverage directory used by tools like istanbul
+# Coverage reports are used for local dev and test by tools like:
+# Jest (JavaScript testing framework), Istanbul/nyc (JavaScript/Node.js) and
+# Linux Test Project LCOV graphical front-end displaying code coverage info.
 coverage/
-*.lcov
-
-# nyc test coverage
 .nyc_output
+*.lcov
 
 # Dependency directories
 node_modules/

.gitmodules

@@ -18,7 +18,7 @@
 	url = https://github.com/modelearth/comparison
 [submodule "team"]
 	path = team
-	url = https://github.com/DreamStudioCode/team.git
+	url = https://github.com/modelearth/team
 [submodule "realitystream"]
 	path = realitystream
 	url = https://github.com/modelearth/realitystream
@@ -28,3 +28,24 @@
 [submodule "cloud"]
 	path = cloud
 	url = https://github.com/modelearth/cloud
+[submodule "codechat"]
+	path = codechat
+	url = https://github.com/modelearth/codechat
+[submodule "exiobase"]
+	path = exiobase
+	url = https://github.com/modelearth/exiobase
+[submodule "io"]
+	path = io
+	url = https://github.com/modelearth/io
+[submodule "profile"]
+	path = profile
+	url = https://github.com/modelearth/profile
+[submodule "reports"]
+	path = reports
+	url = https://github.com/modelearth/reports
+[submodule "community-forecasting"]
+	path = community-forecasting
+	url = https://github.com/modelearth/community-forecasting
+[submodule "products-data"]
+	path = products-data
+	url = https://github.com/modelearth/products-data