Added Notebook LM CRM PRD from BenN

.claude/settings.local.json

@@ -2,6 +2,7 @@
   "permissions": {
     "allow": [
       "Bash(python:*)",
+      "Bash(python3:*)",
       "Bash(grep:*)",
       "Bash(find:*)",
       "Bash(rg:*)",
@@ -25,10 +26,33 @@
       "Bash(yarn build)",
       "Bash(git rm:*)",
       "Bash(git checkout:*)",
-      "Bash(awk:*)"
+      "Bash(awk:*)",
+      "Bash(node:*)",
+      "Bash(lsof:*)",
+      "Bash(xargs kill:*)",
+      "Bash(cat:*)",
+      "WebSearch",
+      "Bash(cargo check:*)",
+      "Bash(cargo build:*)",
+      "Bash(claude:*)",
+      "Bash(npm update:*)",
+      "Bash(npm audit:*)",
+      "Bash(npm install:*)",
+      "Bash(npm run build:*)",
+      "WebFetch(domain:raw.githubusercontent.com)",
+      "WebFetch(domain:www.ars.usda.gov)",
+      "Bash(for dir in cv/*/)",
+      "Bash(do [ -f \"$dir/index.html\" ])",
+      "Bash(done)",
+      "Bash(while read dir)",
+      "Bash(do [ -d \"cv/$dir\" ]:*)",
+      "Bash([ -f \"cv/$dir/detailed.json\" ])",
+      "Bash([ -d \"cv/$dir\" ])",
+      "Bash(git config:*)",
+      "WebFetch(domain:en.wikipedia.org)"
     ],
     "deny": [
       "Read(../**)"
     ]
   }
-}
+}

.dockerignore

@@ -0,0 +1,164 @@
+# Git files and directories
+.git
+.gitignore
+.gitmodules
+.gitattributes
+.github/
+
+# Docker files (avoid copying Docker config into the image)
+Dockerfile
+docker-compose.yml
+docker-compose*.yml
+.dockerignore
+docker/
+
+# CI/CD and tooling
+.claude/
+.vscode/
+.idea/
+
+# Extra repos that are not submodules
+apps
+community
+community-data
+community-timelines
+data-commons
+data-pipeline
+display
+explore
+membersense
+modelearth.github.io
+nisar
+panels
+products-big-files-to-delete
+reality.streamlit*
+recycling
+requests
+resources
+trade-data
+topojson
+useeio
+useeio-json
+useeio.js
+wiki
+
+# Configuration files
+config
+config/settings.js
+config/keys.json
+config/.env
+settings.local.json
+.siterepos
+
+# Auth keys and secrets
+*.key
+*.pem
+env/*
+.env
+.env.*
+!.env.example
+*.env
+
+# Logs
+*.log
+logs/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg
+*.egg-info/
+dist/
+build/
+*.whl
+pip-log.txt
+pip-delete-this-directory.txt
+.pytest_cache/
+.mypy_cache/
+.dmypy.json
+dmypy.json
+venv/
+env/
+ENV/
+.venv
+
+# Testing and coverage
+coverage/
+.nyc_output
+*.lcov
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+nosetests.xml
+coverage.xml
+*.cover
+
+# Dependencies (install fresh in Docker)
+node_modules/
+bower_components/
+
+# Build outputs and cache
+.next/
+.nuxt/
+.cache/
+.parcel-cache/
+.vuepress/dist/
+out/
+dist/
+build/
+target/
+
+# Temporary files
+*.tmp
+*.temp
+*.swp
+*.swo
+*~
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+Desktop.ini
+
+# IDE files
+*.sublime-project
+*.sublime-workspace
+.idea/
+*.iml
+.vscode/
+*.code-workspace
+
+# Optional npm/yarn files
+.npm
+.yarn-integrity
+.pnp.*
+.node_repl_history
+
+# Serverless
+.serverless/
+
+# Other
+.nojekyll
+*.tgz
+*.bak
+*.backup
+codechat-vectors/
+grid/

.env.example.old

@@ -0,0 +1,9 @@
+# Ideally not using the following since we have 3 databases: COMMONS, EXIOBASE, LOCATIONS
+
+# Environment variables for Docker Compose
+# These are used for variable substitution in docker-compose.yml
+
+# PostgreSQL credentials (used for POSTGRES_USER and POSTGRES_PASSWORD)
+POSTGRES_DEFAULT_USER=postgres
+POSTGRES_DEFAULT_PASSWORD=yourpassword
+POSTGRES_DEFAULT_DB=postgres

.github/workflows/vector_db_sync.yml

@@ -0,0 +1,62 @@
+name: VectorDB Sync
+
+on:
+  pull_request:
+    types: [closed]
+  push:
+    branches: [ main ]
+
+jobs:
+  sync:
+    if: |
+      (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo with submodules
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r chat/ingestion/requirements.txt
+
+      - name: Run VectorDB Sync
+        env:
+          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+          VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
+        run: python chat/ingestion/vector_db_sync.py --skip-on-missing-keys
+
+      - name: Error summary
+        if: always()
+        shell: bash
+        run: |
+          if [ -f chat/.vector_sync_errors.jsonl ]; then
+            {
+              echo "## Vector Sync Errors";
+              echo;
+              echo '```jsonl';
+              tail -n 200 chat/.vector_sync_errors.jsonl;
+              echo '```';
+            } >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "No errors file present."
+          fi
+
+      - name: Upload VectorDB Sync errors
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: vector-sync-errors
+          path: chat/.vector_sync_errors.jsonl
+          if-no-files-found: ignore
+          retention-days: 7