diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 7b76302a6..22e7242e6 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -21,7 +21,11 @@ "Bash(rm:*)", "Bash(curl:*)", "Bash(gh:*)", - "Bash(./git.sh:*)" + "Bash(./git.sh:*)", + "Bash(yarn build)", + "Bash(git rm:*)", + "Bash(git checkout:*)", + "Bash(awk:*)" ], "deny": [ "Read(../**)" diff --git a/.github/workflows/vector_sync.yml b/.github/workflows/vector_sync.yml index 710a8fa84..3e2f6f286 100644 --- a/.github/workflows/vector_sync.yml +++ b/.github/workflows/vector_sync.yml @@ -1,18 +1,13 @@ name: VectorDB Sync on: - # Run when a PR is merged into main pull_request: types: [closed] - - # Run on direct pushes to main branch push: - branches: - - main + branches: [ main ] jobs: sync: - # For PR: only if merged into main if: | (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || (github.event_name == 'push' && github.ref == 'refs/heads/main') @@ -33,45 +28,38 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt || true + # Try repo-specific requirements + pip install -r codechat/requirements.txt || true + # Core sync deps (serverless Pinecone SDK) + pip install --upgrade tiktoken tree_sitter tqdm pyyaml pandas pinecone openai - - name: Detect changed files (root + submodules) + - name: Run VectorDB Sync (commit-range) + env: + PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + shell: bash run: | - echo "🔎 Detecting changes..." - if [ "${{ github.event_name }}" == "push" ]; then - echo "Push to main branch detected" - git diff --name-status HEAD^ HEAD > changed_files.txt || true - else - echo "Pull request merged into main detected" - git diff --name-status origin/main HEAD > changed_files.txt || true + set -euo pipefail + # Skip if sync script not present (e.g., codechat PR not merged yet) + if [ ! -f codechat/vectordb_sync.py ]; then + echo "[skip] codechat/vectordb_sync.py not found; skipping VectorDB sync." + exit 0 fi - cat changed_files.txt - - echo "🔎 Detecting submodule pointer changes..." - git diff --submodule > submodule_changes.txt || true - cat submodule_changes.txt - - echo "🔎 Expanding submodule changes into file-level diffs..." - while read -r line; do - submodule=$(echo "$line" | awk '{print $1}') - oldsha=$(echo "$line" | awk '{print $2}') - newsha=$(echo "$line" | awk '{print $3}') - if [ -n "$submodule" ] && [ -n "$oldsha" ] && [ -n "$newsha" ]; then - echo "📂 Submodule: $submodule ($oldsha → $newsha)" - pushd $submodule > /dev/null - git fetch origin $oldsha $newsha || true - git diff --name-status $oldsha $newsha | sed "s|^|$submodule/|" >> ../changed_files.txt - popd > /dev/null - fi - done < <(git diff --submodule | grep Submodule) - - echo "✅ Final changed files list:" - cat changed_files.txt - -# - name: Run VectorDB Sync -# env: -# PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} -# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} -# run: | -# python scripts/sync_pinecone.py changed_files.txt \ No newline at end of file + # Skip if required secrets not provided + if [ -z "${PINECONE_API_KEY:-}" ] || [ -z "${OPENAI_API_KEY:-}" ]; then + echo "[skip] Missing PINECONE_API_KEY or OPENAI_API_KEY; skipping VectorDB sync." + exit 0 + fi + if [ "${{ github.event_name }}" = "push" ]; then + BASE="${{ github.event.before }}"; HEAD="${{ github.sha }}" + else + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.merge_commit_sha }}" + if [ -z "$HEAD" ]; then HEAD="${{ github.sha }}"; fi + fi + if [ "$BASE" = "0000000000000000000000000000000000000000" ] || [ -z "$BASE" ]; then + BASE="${HEAD}^" + fi + echo "[info] Sync range: $BASE..$HEAD" + python codechat/vectordb_sync.py --from-commit "$BASE" --to-commit "$HEAD" --repo-root . diff --git a/.gitignore b/.gitignore index 37b38f879..51c2d4759 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,27 @@ -# Repos that are not submodules -profile -exiobase -trade-data -useeio.js -io -codechat -membersense -data-commons -data-pipeline +# Extra repos that are not submodules +apps +community community-data community-timelines +data-commons +data-pipeline +display +explore +membersense +nisar +panels +products-big-files-to-delete +reality.streamlit* +requests +resources +trade-data +topojson +useeio +useeio-json +useeio.js +wiki -# Configuration files with sensitive data +# Configuration files config/settings.js config/keys.json config/.env @@ -34,13 +44,14 @@ pids *.pid *.seed *.pid.lock +__pycache__ -# Coverage directory used by tools like istanbul +# Coverage reports are used for local dev and test by tools like: +# Jest (JavaScript testing framework), Istanbul/nyc (JavaScript/Node.js) and +# Linux Test Project LCOV graphical front-end displaying code coverage info. coverage/ -*.lcov - -# nyc test coverage .nyc_output +*.lcov # Dependency directories node_modules/ diff --git a/.gitmodules b/.gitmodules index 7305fcf46..5e3fa3d65 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,7 +18,7 @@ url = https://github.com/modelearth/comparison [submodule "team"] path = team - url = https://github.com/DreamStudioCode/team.git + url = https://github.com/modelearth/team [submodule "realitystream"] path = realitystream url = https://github.com/modelearth/realitystream @@ -28,3 +28,24 @@ [submodule "cloud"] path = cloud url = https://github.com/modelearth/cloud +[submodule "codechat"] + path = codechat + url = https://github.com/modelearth/codechat +[submodule "exiobase"] + path = exiobase + url = https://github.com/modelearth/exiobase +[submodule "io"] + path = io + url = https://github.com/modelearth/io +[submodule "profile"] + path = profile + url = https://github.com/modelearth/profile +[submodule "reports"] + path = reports + url = https://github.com/modelearth/reports +[submodule "community-forecasting"] + path = community-forecasting + url = https://github.com/modelearth/community-forecasting +[submodule "products-data"] + path = products-data + url = https://github.com/modelearth/products-data diff --git a/CLAUDE.md b/CLAUDE.md index a57b4bc1d..8a2cc82bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,564 +1,3 @@ # CLAUDE.md -## Start server or Restart server - -### Start Server -When you type "start server", run - -```bash -nohup python -m http.server 8887 > /dev/null 2>&1 & -``` - -Note: Uses nohup to run server in background and redirect output to avoid timeout. - - -### Start Rust API Server -When you type "start rust", change to the team submodule directory in the repository root and run - -```bash -cd team -# Ensure Rust is installed and cargo is in PATH -source ~/.cargo/env 2>/dev/null || echo "Install Rust first: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" -# Copy .env.example to .env only if .env doesn't exist -[ ! -f .env ] && cp .env.example .env -# Start the server with correct binary name -nohup cargo run --bin partner_tools -- serve > server.log 2>&1 & -``` - -Note: The team repository is a submodule located in the repository root directory. The Rust API server runs on port 8081. Requires Rust/Cargo to be installed on the system. The .env file is created from .env.example only if it doesn't already exist. - -### Update submodules: -When you type "update submodules", run -```bash -# Navigate to webroot first -cd $(git rev-parse --show-toplevel) -git submodule update --remote --recursive -``` - -### Commit submodules: -When you type "commit submodules", run -```bash -./git.sh commit submodules [nopr] -``` -**Note**: This commits all submodules with changes AND updates the webroot parent repository with new submodule references. Includes automatic PR creation on push failures. - -### PR [submodule name]: -Create a pull request for a submodule when you lack collaborator privileges: -```bash -cd [submodule name] -git add . && git commit -m "Description of changes" -git push origin feature-branch-name -gh pr create --title "Update [submodule name]" --body "Description of changes" -cd .. -``` - -## IMPORTANT: Git Commit Policy - -**NEVER commit changes without explicit user request.** - -- Only run git commands (add, commit, push) when the user specifically says "commit" or directly requests it -- After making code changes, STOP and wait for user instruction -- Build and test changes as needed, but do not commit automatically -- The user controls when changes are committed to the repository - -## Comprehensive Update Command - -### Update -When you type "Update", run this comprehensive update workflow that pulls from all parent repos, updates submodules and forks, and prompts for pushes: - -```bash -./git.sh update -``` - -All complex git operations are now handled by the git.sh script to avoid shell parsing issues. - -### GitHub Account Management -The git.sh script automatically detects the current GitHub CLI user and adapts accordingly: - -```bash -gh auth logout # Log out of current GitHub account -gh auth login # Log into different GitHub account -./git.sh auth # Refresh git credentials and update all remotes -``` - -When you switch GitHub accounts, the script will: -- **Automatically detect** the new user during commit/update operations -- **Clear cached git credentials** from previous account -- **Refresh authentication** to use new GitHub CLI credentials -- **Update remote URLs** to point to the new user's forks -- **Create PRs** from the new user's account -- **Fork repositories** to the new user's account when needed - -**Automatic Credential Management:** -- Detects when GitHub user has changed since last run -- Clears cached credentials from credential manager and macOS keychain -- Runs `gh auth setup-git` to sync git with GitHub CLI -- Prevents permission denied errors from stale credentials - -**Update Command Features:** -- **Pull from Parents**: Updates webroot, submodules, and trade repos from their respective ModelEarth parent repositories -- **Fork-Aware**: Automatically adds upstream remotes for parent repos when working with forks -- **Partnertools Exclusion**: Completely skips any repositories associated with partnertools GitHub account -- **Merge Strategy**: Uses automatic merge with no-edit to incorporate upstream changes -- **Conflict Handling**: Reports merge conflicts for manual resolution when they occur -- **Status Reporting**: Provides clear feedback on what was updated and any issues encountered -- **Push Guidance**: Prompts user with specific commands for pushing changes back to forks and parent repos -- **Comprehensive Workflow**: Handles webroot, all submodules, and all trade repositories in one command - -**Post-Update Recommendations:** -After running "./git.sh update", review changes and use these commands as needed: -- `./git.sh commit` - Push all changes (webroot + submodules + forks) with PR creation -- `./git.sh commit submodules` - Push only submodule changes -- `./git.sh commit [specific-name]` - Push changes for a specific repository - -**Git.sh Usage:** -```bash -./git.sh update # Full update workflow -./git.sh commit # Complete commit: webroot, all submodules, and trade repos -./git.sh commit [name] # Commit specific submodule -./git.sh commit submodules # Commit all submodules only -./git.sh commit [name] nopr # Skip PR creation on push failures -``` - - -## Submodule Management - -This repository contains the following git submodules configured in `.gitmodules`: -- **localsite** - https://github.com/ModelEarth/localsite -- **feed** - https://github.com/modelearth/feed -- **swiper** - https://github.com/modelearth/swiper -- **home** - https://github.com/ModelEarth/home -- **products** - https://github.com/modelearth/products -- **comparison** - https://github.com/modelearth/comparison -- **team** - https://github.com/modelearth/team -- **projects** - https://github.com/modelearth/projects -- **realitystream** - https://github.com/modelearth/realitystream -- **cloud** - https://github.com/modelearth/cloud - -**IMPORTANT**: All directories listed above are git submodules, not regular directories. They appear as regular directories when browsing but are actually git submodule references. Always treat them as submodules in git operations. - -### Repository Root Navigation -**CRITICAL**: Always ensure you're in the webroot repository before executing any commands. The CLI session is pointed to the webroot directory, and all operations must start from there: - -```bash -# ALWAYS navigate to webroot repository root first (required for all operations) -cd $(git rev-parse --show-toplevel) - -# Verify you're in the correct webroot repository -git remote -v -# Should show: origin https://github.com/ModelEarth/webroot.git - -# If git rev-parse returns the wrong repository (submodule/trade repo), manually navigate to webroot -# Use your system's webroot path, never hardcode paths in documentation -``` - -**IMPORTANT FILE PATH POLICY**: -- **NEVER hardcode specific file paths** from any user's computer in code or documentation -- **NEVER include paths like `/Users/username/` or `C:\Users\`** in any commands or examples -- Always use relative paths, environment variables, or git commands to determine paths dynamically -- Use `$(git rev-parse --show-toplevel)` when already in the correct repository context -- If `git rev-parse --show-toplevel` returns incorrect paths (submodule/trade repo instead of webroot), the user must manually navigate to their webroot directory using their system's actual path - -**IMPORTANT**: The `git rev-parse --show-toplevel` command returns the top-level directory of whatever git repository you're currently in. If you're inside a submodule or trade repo, it will return that repository's root instead of the webroot. In such cases, you must manually navigate to your actual webroot directory location on your system. - -**Common Issue**: If submodule commands fail or you get "pathspec did not match" errors, you're likely in a submodule directory instead of the webroot. Navigate back to your webroot directory using your system's actual webroot path before running any commands. - -### IMPORTANT: "commit [name]" Command Requirements -When a user says "commit [name]", use the git.sh script: - -```bash -./git.sh commit [name] [nopr] -``` - -The git.sh script handles all the complex logic including: -- Submodule detection and committing -- Automatic PR creation on push failures -- Webroot submodule reference updates -- Support for 'nopr' flag to skip PR creation - -**Direct Commit Method (if foreach strategy fails):** - -Used when the `git submodule foreach` strategy fails, such as: -- **Detached HEAD state**: Submodule is not on a proper branch -- **Corrupted submodule**: `.git` folder or configuration is damaged -- **Branch conflicts**: Submodule is on a different branch than expected -- **Nested submodules**: Complex submodule hierarchies that confuse foreach -- **Permission issues**: File system permissions prevent git operations within submodules - -**⚠️ IMPORTANT**: Do not initialize new submodules unless explicitly requested by the user. If a directory exists but is not properly initialized as a submodule, treat it as a standalone repository or ignore it rather than converting it to a submodule. - -```bash -# Step 0: ALWAYS start from webroot -cd $(git rev-parse --show-toplevel) - -# Direct submodule commit (when foreach method doesn't work) -cd [submodule name] -git checkout main # Ensure on main branch (fixes detached HEAD) -git add . && git commit -m "Description of changes" -if git push origin main; then - echo "✅ Successfully pushed [submodule name] submodule" -elif [ "$SKIP_PR" != "true" ]; then - git push origin HEAD:feature-[submodule name]-direct && gh pr create --title "Update [submodule name] submodule" --body "Direct update of [submodule name] submodule" --base main --head feature-[submodule name]-direct || echo "PR creation failed" - echo "🔄 Created PR for [submodule name] submodule due to permission restrictions" -fi - -# Return to webroot and update submodule reference -cd $(git rev-parse --show-toplevel) -git submodule update --remote [submodule name] -git add [submodule name] -git commit -m "Update [submodule name] submodule" -if git push; then - echo "✅ Successfully updated [submodule name] submodule reference" -elif [ "$SKIP_PR" != "true" ]; then - git push origin HEAD:feature-webroot-[submodule name]-ref && gh pr create --title "Update [submodule name] submodule reference" --body "Update submodule reference for [submodule name]" --base main --head feature-webroot-[submodule name]-ref || echo "Webroot PR creation failed" - echo "🔄 Created PR for webroot [submodule name] submodule reference" -fi -``` - -**⚠️ CRITICAL**: -- **NEW**: Automatic PR creation when push permissions are denied -- **NEW**: 'nopr' or 'No PR' (case insensitive) flag to skip PR creation -- **NEW**: All commit commands include PR fallback for permission failures -- **NEW**: Intelligent fallback strategy handles unrecognized names gracefully -- **NEW**: Three-tier approach: submodule → standalone repo → webroot fallback -- **NEW**: Always checks for actual changes before committing -- **NEW**: Provides clear success/failure feedback with ✅ and 🔄 indicators -- **NEVER initialize new submodules unless explicitly requested by user** -- **NEVER convert existing directories to submodules automatically** -- Method 1 handles detached HEAD states automatically -- Both methods require updating the parent repository -- If git submodule foreach fails, the submodule may not exist or be corrupted -- Always check status first to see which submodules actually have changes -- Use conditional `if [ "$name" = "submodule" ]` to target specific submodule and avoid "nothing to commit" errors from clean submodules -- The `--recursive` flag ensures nested submodules are handled properly -- Requires GitHub CLI (gh) for PR creation functionality - -### Quick Commands for Repositories -- **"commit [name] [nopr]"**: Intelligent commit with PR fallback - tries submodule → standalone repo → webroot fallback -- **"push [submodule name]"**: Only push submodule changes (steps 1-3) -- **"PR [submodule name]"**: Create pull request workflow -- **"commit submodules [nopr]"**: Commit all submodules with PR fallback when push fails -- **"commit forks [nopr]"**: Commit all trade repo forks and create PRs to parent repos -- **"commit [nopr]"**: Complete commit workflow with PR fallback - commits webroot, all submodules, and all forks - -**PR Fallback Behavior**: All commit commands automatically create pull requests when direct push fails due to permission restrictions. Add 'nopr' or 'No PR' (case insensitive) at the end of any commit command to skip PR creation. - -When displaying "Issue Resolved" use the same checkbox icon as "Successfully Updated" - -### Additional Notes -- Allow up to 12 minutes to pull repos (large repositories) -- Always verify both submodule AND parent repository are updated - -## Git Commit Guidelines -- **NEVER add Claude Code attribution or co-authored-by lines to commits** -- **NEVER add "🤖 Generated with [Claude Code]" or similar footers** -- Keep commit messages clean and focused on the actual changes -- Include a brief summary of changes in the commit text - -## Quick Commands - -When you type "restart", run this single command to restart the server in seconds: -```bash -cd $(git rev-parse --show-toplevel) && pkill -f "node.*index.js"; (cd server && NODE_ENV=production nohup node index.js > /dev/null 2>&1 &) -``` - -When you type "quick", add the following permissions block to setting.local.json under allow. " -When you type "confirm" or "less quick", remove it: -```json -[ - "Bash(yarn setup)", - "Bash(npx update-browserslist-db:*)", - "Bash(mkdir:*)", - "Bash(yarn build)", - "Bash(cp:*)", - "Bash(npx prisma generate:*)", - "Bash(npx prisma migrate:*)", - "Bash(pkill:*)", - "Bash(curl:*)", - "Bash(git submodule add:*)", - "Bash(rm:*)", - "Bash(find:*)", - "Bash(ls:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(git push:*)" -] -``` - -## Trade Repositories - -### Trade Repo List -The following trade repositories are used for multi-regional input-output (MRIO) analysis: -- **exiobase** - https://github.com/modelearth/exiobase -- **profile** - https://github.com/modelearth/profile -- **useeio.js** - https://github.com/modelearth/useeio.js -- **io** - https://github.com/modelearth/io - -**IMPORTANT**: These trade repos are cloned to the webroot root directory, not as submodules, since typical sites only use trade output via the existing comparison submodule. - -### Fork Trade Repos - -```bash -fork trade repos to [your github account] -``` - -The above runs these commands: -```bash -# Fork repositories using GitHub CLI (requires 'gh' to be installed and authenticated) -gh repo fork modelearth/exiobase --clone=false -gh repo fork modelearth/profile --clone=false -gh repo fork modelearth/useeio.js --clone=false -gh repo fork modelearth/io --clone=false -``` - -### Clone Trade Repos - -```bash -clone trade repos from [your github account] -``` - -The above runs these commands: -```bash -# Navigate to webroot repository root first -cd $(git rev-parse --show-toplevel) - -# Clone trade repos to webroot root -git clone https://github.com/[your github account]/exiobase exiobase -git clone https://github.com/[your github account]/profile profile -git clone https://github.com/[your github account]/useeio.js useeio.js -git clone https://github.com/[your github account]/io io -``` - -### Commit All Submodules - -```bash -commit submodules [nopr] -``` - -The above commits changes to all submodules that have uncommitted changes: -```bash -# Navigate to webroot repository root first and detect no-PR flag -cd $(git rev-parse --show-toplevel) -SKIP_PR=false -if [[ "$*" =~ (nopr|no\ pr)$ ]] || [[ "$*" =~ (NOPR|NO\ PR)$ ]]; then - SKIP_PR=true -fi - -# Check which submodules have changes first -git submodule foreach 'git status' - -# Commit all submodules that have changes with PR fallback -git submodule foreach --recursive ' - if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update $name submodule" - if git push origin HEAD:main; then - echo "✅ Successfully pushed $name submodule" - elif [ "$SKIP_PR" != "true" ]; then - git push origin HEAD:feature-$name-updates && gh pr create --title "Update $name submodule" --body "Automated submodule update from webroot" --base main --head feature-$name-updates || echo "PR creation failed for $name" - echo "🔄 Created PR for $name submodule due to permission restrictions" - fi - fi -' - -# Update parent repository with submodule references -git submodule update --remote -if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update submodule references" - if git push; then - echo "✅ Successfully updated webroot submodule references" - elif [ "$SKIP_PR" != "true" ]; then - git push origin HEAD:feature-webroot-submodule-updates && gh pr create --title "Update submodule references" --body "Automated update of all submodule references" --base main --head feature-webroot-submodule-updates || echo "Webroot PR creation failed" - echo "🔄 Created PR for webroot submodule references due to permission restrictions" - fi -fi -``` - -### Commit Trade Repo Forks - -```bash -commit forks [nopr] -``` - -The above commits changes to all trade repo forks and creates pull requests to their parent repositories: -```bash -# Navigate to webroot repository root first and detect no-PR flag -cd $(git rev-parse --show-toplevel) -SKIP_PR=false -if [[ "$*" =~ (nopr|no\ pr)$ ]] || [[ "$*" =~ (NOPR|NO\ PR)$ ]]; then - SKIP_PR=true -fi - -# Check each trade repo for changes and create PRs -for repo in exiobase profile useeio.js io; do - if [ -d "$repo" ]; then - cd "$repo" - if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update $repo repository" - if git push origin main; then - echo "✅ Successfully pushed $repo repository" - else - echo "⚠️ Push failed for $repo repository" - fi - - # Only create PR for forks (when you don't have direct push access to parent) - if [ "$SKIP_PR" != "true" ]; then - # Check if this is a fork by comparing remote URL with expected parent - REMOTE_URL=$(git remote get-url origin) - if [[ "$REMOTE_URL" =~ "modelearth/$repo" ]] && [[ "$REMOTE_URL" != *"ModelEarth/$repo"* ]]; then - gh pr create --title "Update $repo" --body "Automated update from webroot integration" --base main --head main || echo "PR creation failed for $repo" - echo "🔄 Created PR for $repo fork to parent repository" - else - echo "✅ Direct push succeeded - no PR needed for $repo" - fi - fi - fi - cd .. - fi -done -``` - -**Note**: This command requires GitHub CLI (gh) to be installed and authenticated for PR creation. It will create PRs for all trade repo forks unless 'nopr' is specified. Use 'commit forks nopr' to skip PR creation. - -**Common Issues:** -- **Repository moved errors**: Update remote URLs if you see "This repository moved" messages: - ```bash - cd [repo-name] - git remote set-url origin [new-url] - ``` -- **GitHub CLI authentication**: Run `gh auth login` to authenticate with GitHub before using PR features - -### Complete Commit Workflow - -```bash -commit [nopr] -``` - -The above runs a comprehensive commit workflow that handles webroot, all submodules, and all trade repo forks with automatic PR creation: - -```bash -# Navigate to webroot repository root first and detect no-PR flag -cd $(git rev-parse --show-toplevel) -SKIP_PR=false -if [[ "$*" =~ (nopr|no\ pr)$ ]] || [[ "$*" =~ (NOPR|NO\ PR)$ ]]; then - SKIP_PR=true -fi - -# Step 1: Commit webroot repository changes -if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update webroot repository" - if git push; then - echo "✅ Successfully committed webroot repository" - elif [ "$SKIP_PR" != "true" ]; then - git push origin HEAD:feature-webroot-comprehensive-update && gh pr create --title "Comprehensive webroot update" --body "Automated comprehensive update of webroot repository" --base main --head feature-webroot-comprehensive-update || echo "Webroot PR creation failed" - echo "🔄 Created PR for webroot repository due to permission restrictions" - fi -fi - -# Step 2: Commit all submodules that have changes -git submodule foreach ' - if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update $name submodule" - if git push origin HEAD:main; then - echo "✅ Successfully pushed $name submodule" - elif [ "$SKIP_PR" != "true" ]; then - git push origin HEAD:feature-$name-comprehensive-updates && gh pr create --title "Update $name submodule" --body "Automated submodule update from comprehensive commit" --base main --head feature-$name-comprehensive-updates || echo "PR creation failed for $name" - echo "🔄 Created PR for $name submodule due to permission restrictions" - fi - fi -' - -# Step 3: Update parent repository with submodule references -git submodule update --remote -if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update submodule references" - if git push; then - echo "✅ Successfully updated webroot submodule references" - elif [ "$SKIP_PR" != "true" ]; then - git push origin HEAD:feature-webroot-submodule-comprehensive && gh pr create --title "Update all submodule references" --body "Comprehensive update of all submodule references" --base main --head feature-webroot-submodule-comprehensive || echo "Submodule reference PR creation failed" - echo "🔄 Created PR for webroot submodule references due to permission restrictions" - fi -fi - -# Step 4: Commit trade repo forks and create PRs -for repo in exiobase profile useeio.js io; do - if [ -d "$repo" ]; then - cd "$repo" - if [ -n "$(git status --porcelain)" ]; then - git add . - git commit -m "Update $repo repository" - if git push origin main; then - echo "✅ Successfully pushed $repo repository" - else - echo "⚠️ Push failed for $repo repository" - fi - - # Only create PR for forks (when you don't have direct push access to parent) - if [ "$SKIP_PR" != "true" ]; then - # Check if this is a fork by comparing remote URL with expected parent - REMOTE_URL=$(git remote get-url origin) - if [[ "$REMOTE_URL" =~ "modelearth/$repo" ]] && [[ "$REMOTE_URL" != *"ModelEarth/$repo"* ]]; then - gh pr create --title "Update $repo" --body "Automated update from comprehensive webroot commit" --base main --head main || echo "PR creation failed for $repo" - echo "🔄 Created PR for $repo fork to parent repository" - else - echo "✅ Direct push succeeded - no PR needed for $repo" - fi - fi - fi - cd .. - fi -done -``` - -**Note**: This is the most comprehensive commit command that handles all repository types in the webroot ecosystem with automatic PR fallback when push permissions are denied. Use 'commit nopr' to skip all PR creation. It will only process repositories that have actual changes. - - -### Background Development Rust Server resides in "team" submodule folder (ALWAYS USE THIS) - -**IMPORTANT**: All Rust development occurs in the `team/` submodule directory. Always navigate to `team/` before running Rust commands. - -```bash -# Navigate to team directory first -cd team - -# ALWAYS use this command to start server - keeps running in background -nohup cargo run serve > server.log 2>&1 & - -# Check if dev server is running -curl http://localhost:8081/api/health - -# Stop dev background server -lsof -ti:8081 | xargs kill -9 -``` - -#### Rust Development Commands (run from team/ directory) -```bash -# Build and Run -cargo build # Build the project -cargo run -- serve # Start REST API server (blocks terminal - not recommended) -cargo run -- init-db # Initialize database schema -cargo check # Check code without building -cargo clippy # Run linting -cargo test # Run tests - -# Database Management -cargo run -- init-db # Create all tables with relationships and constraints -``` - -#### Environment Configuration -- Server host/port configurable via `SERVER_HOST`/`SERVER_PORT` environment variables -- **Primary Database**: PostgreSQL (COMMONS_HOST in .env file) -- **Trade Flow Database**: PostgreSQL (EXIOBASE_HOST in .env file) -- Store auth keys in separate config file excluded by `.gitignore` - -#### Alternative Commands (NOT RECOMMENDED) -- `cargo run serve` - Blocks terminal, stops when you exit (DO NOT USE) -- `cargo run -- serve` - Same issue, blocks terminal (DO NOT USE) - -#### Project Type -This is a **project posting, assignment and to-do tracking system** - a CRM-style tool for managing public-facing listings with searchable directories, team collaboration, and AI integration using Gemini. +The CLAUDE.md instructions reside in team/CLAUDE.md \ No newline at end of file diff --git a/README.md b/README.md index 5527cdff7..b3545fe5a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,13 @@ Then view pages at: [localhost:8887/home](http://localhost:8887/home/) [localhost:8887/feed](http://localhost:8887/feed/) +
+ + 🗄️ + Rust API and Database + +
+ Look at the code in your webroot with an editor like [Sublime Text](https://www.sublimetext.com/) ($99), [VS Code](https://code.visualstudio.com/) or [WebStorm](https://www.jetbrains.com/webstorm/).
@@ -25,18 +32,31 @@ Look at the code in your webroot with an editor like [Sublime Text](https://www. Update and make commits often (at least hourly). Append nopr" or "No PR" if you are not yet ready to send a Pull Request. -Run "update" hourly to safely pull updates from the ModelEarth parent repositories. +Run "pull" hourly to safely pull updates to the modelearth repos residing in your webroot + +When making any change, run "push" to send a PR. +"push" updates the webroot, submodules and forks. It does a "pull" automatically first. + + push + +If you find "push" is asking for mulitple approvals, Claude may not have read the claude.md instructions. +For the first usage, include extra guidance: + + push using claude.md with git.sh + +Addtional deployment commands: + + push [folder name] # Deploy a specific submodule or fork + push submodules # Deploy changes in all submodules + push forks # Deploy the extra forks added -When making any change, run "commit" to send a PR. This will also commit changes in submodules and forks. - update - commit +## Alternative to using Claude Code CLI -Addtional options: +You can also use Github Desktop to choose a repo inthe webroot using "File > Add Local Repository". +Then submit a PR through the Github.com website. (The "push" with Claude will send a PR automatically.) - commit [folder name] # commit a specific submodule or fork - commit submodules # commit changes from all submodules changed - commit forks # commit the 4 forks added for the trade flow +You can refresh all your local submodule by running: -Or when changing a cloned repo, commit the specific repo using Github Desktop. Then submit a PR through the Github.com website. \ No newline at end of file + git submodule foreach 'git pull origin main || git pull origin master' \ No newline at end of file diff --git a/cloud b/cloud index 189c555c1..2fd442610 160000 --- a/cloud +++ b/cloud @@ -1 +1 @@ -Subproject commit 189c555c15cf0b959ca945fa807335f50559d91d +Subproject commit 2fd442610008ddc0ae3e00dbeb3adc91682a2d5e diff --git a/codechat b/codechat new file mode 160000 index 000000000..0095a9210 --- /dev/null +++ b/codechat @@ -0,0 +1 @@ +Subproject commit 0095a9210c93a4b844fe97c6acf7926472768e44 diff --git a/community-forecasting b/community-forecasting new file mode 160000 index 000000000..d6361562d --- /dev/null +++ b/community-forecasting @@ -0,0 +1 @@ +Subproject commit d6361562dbba3fa8b878841cc0c0720862a447a9 diff --git a/comparison b/comparison index 5e29dbfc9..46372e22a 160000 --- a/comparison +++ b/comparison @@ -1 +1 @@ -Subproject commit 5e29dbfc9ebfe9f5533fdcfa436b7aedd163123a +Subproject commit 46372e22a2c9f5019a366120e2644f20bafcc74d diff --git a/exiobase b/exiobase new file mode 160000 index 000000000..464536e46 --- /dev/null +++ b/exiobase @@ -0,0 +1 @@ +Subproject commit 464536e461a7fdc54578ab4c4c3a3df00061adb7 diff --git a/favicon.ico b/favicon.ico new file mode 100644 index 000000000..ad6627f90 Binary files /dev/null and b/favicon.ico differ diff --git a/feed b/feed index ab72c3fad..c97995602 160000 --- a/feed +++ b/feed @@ -1 +1 @@ -Subproject commit ab72c3fad8a4e22af099d94da8b49d3bb9b458fe +Subproject commit c979956025104ac5cbab47a820f7825b835b6e44 diff --git a/git.sh b/git.sh index 9dce03556..3dda9dcea 100755 --- a/git.sh +++ b/git.sh @@ -2,6 +2,7 @@ # git.sh - Streamlined git operations for webroot repository # Usage: ./git.sh [command] [options] +# Push commands automatically pull first unless 'nopull' or 'no pull' is specified set -e # Exit on any error @@ -38,9 +39,6 @@ merge_upstream() { return 0 elif git merge upstream/master --no-edit 2>/dev/null; then return 0 - # Only try dev branch for useeio.js - elif [[ "$repo_name" == "useeio.js" ]] && git merge upstream/dev --no-edit 2>/dev/null; then - return 0 else echo "⚠️ Merge conflicts - manual resolution needed" return 1 @@ -365,9 +363,6 @@ commit_push() { # Determine target branch local target_branch="main" - if [[ "$name" == "useeio.js" ]]; then - target_branch="dev" - fi # Check if user owns the repository if is_repo_owner "$name"; then @@ -461,7 +456,7 @@ commit_push() { fi # Update webroot submodule reference if this is a submodule - if [[ "$name" != "webroot" ]] && [[ "$name" != "exiobase" ]] && [[ "$name" != "profile" ]] && [[ "$name" != "useeio.js" ]] && [[ "$name" != "io" ]]; then + if [[ "$name" != "webroot" ]] && [[ "$name" != "exiobase" ]] && [[ "$name" != "profile" ]] && [[ "$name" != "io" ]]; then update_webroot_submodule_reference "$name" "$commit_hash" fi else @@ -477,15 +472,23 @@ commit_push() { fi } -# Update command - streamlined update workflow -update_command() { - echo "🔄 Starting update workflow..." +# Pull command - streamlined pull workflow +pull_command() { + local repo_name="$1" + + echo "🔄 Starting pull workflow..." cd $(git rev-parse --show-toplevel) check_webroot - # Update webroot - echo "📥 Updating webroot..." - git pull origin main 2>/dev/null || echo "⚠️ Pull conflicts in webroot" + # If specific repo name provided, pull only that repo + if [ -n "$repo_name" ]; then + pull_specific_repo "$repo_name" + return + fi + + # Pull webroot + echo "📥 Pulling webroot..." + git pull origin main 2>/dev/null || echo "⚠️ Checking for conflicts in webroot" # Update webroot from parent (skip partnertools) WEBROOT_REMOTE=$(git remote get-url origin) @@ -494,9 +497,9 @@ update_command() { merge_upstream "webroot" fi - # Update submodules - echo "📥 Updating submodules..." - for sub in cloud comparison feed home localsite products projects realitystream swiper team; do + # Pull submodules + echo "📥 Pulling submodules..." + for sub in cloud comparison feed home localsite products projects realitystream swiper team trade codechat exiobase io profile reports community-forecasting; do [ ! -d "$sub" ] && continue cd "$sub" @@ -516,16 +519,16 @@ update_command() { echo "🔄 Updating submodule references..." git submodule update --remote --recursive - # Check for and fix any detached HEAD states after updates - echo "🔍 Checking for detached HEAD states after update..." + # Check for and fix any detached HEAD states after pulls + echo "🔍 Checking for detached HEAD states after pull..." fix_all_detached_heads - # Update trade repos - echo "📥 Updating trade repos..." - for repo in exiobase profile useeio.js io; do + # Pull extra repos + echo "📥 Pulling extra repos..." + for repo in community nisar data-pipeline; do [ ! -d "$repo" ] && continue cd "$repo" - git pull origin main 2>/dev/null || echo "⚠️ Pull conflicts in $repo" + git pull origin main 2>/dev/null || echo "⚠️ Checking for conflicts in $repo" REMOTE=$(git remote get-url origin 2>/dev/null || echo "") if [[ "$REMOTE" != *"partnertools"* ]]; then @@ -535,7 +538,80 @@ update_command() { cd .. done - echo "✅ Update completed! Use: ./git.sh commit" + echo "✅ Pull completed! Use: ./git.sh push" +} + +# Pull specific repository +pull_specific_repo() { + local repo_name="$1" + + cd $(git rev-parse --show-toplevel) + check_webroot + + # Check if it's webroot + if [[ "$repo_name" == "webroot" ]]; then + echo "📥 Pulling webroot..." + git pull origin main 2>/dev/null || echo "⚠️ Checking for conflicts in webroot" + + WEBROOT_REMOTE=$(git remote get-url origin) + if [[ "$WEBROOT_REMOTE" != *"partnertools"* ]]; then + add_upstream "webroot" "true" + merge_upstream "webroot" + fi + echo "✅ Webroot pull completed!" + return + fi + + # Check if it's a submodule + if [[ " cloud comparison feed home localsite products projects realitystream swiper team trade codechat exiobase io profile reports community-forecasting " =~ " $repo_name " ]]; then + if [ -d "$repo_name" ]; then + echo "📥 Pulling submodule: $repo_name..." + cd "$repo_name" + + REMOTE=$(git remote get-url origin 2>/dev/null || echo "") + if [[ "$REMOTE" != *"partnertools"* ]]; then + if [[ "$repo_name" == "localsite" ]] || [[ "$repo_name" == "home" ]]; then + add_upstream "$repo_name" "true" + else + add_upstream "$repo_name" "false" + fi + merge_upstream "$repo_name" + fi + + cd .. + git submodule update --remote "$repo_name" + echo "✅ $repo_name submodule pull completed!" + else + echo "⚠️ Submodule not found: $repo_name" + fi + return + fi + + # Check if it's an extra repo + if [[ " community nisar data-pipeline " =~ " $repo_name " ]]; then + if [ -d "$repo_name" ]; then + echo "📥 Pulling extra repo: $repo_name..." + cd "$repo_name" + git pull origin main 2>/dev/null || echo "⚠️ Checking for conflicts in $repo_name" + + REMOTE=$(git remote get-url origin 2>/dev/null || echo "") + if [[ "$REMOTE" != *"partnertools"* ]]; then + add_upstream "$repo_name" "false" + merge_upstream "$repo_name" + fi + cd .. + echo "✅ $repo_name extra repo pull completed!" + else + echo "⚠️ Extra repo not found: $repo_name" + fi + return + fi + + echo "⚠️ Repository not recognized: $repo_name" + echo "Supported repositories:" + echo " Webroot: webroot" + echo " Submodules: cloud, comparison, feed, home, localsite, products, projects, realitystream, swiper, team, trade, codechat, exiobase, io, profile, reports, community-forecasting" + echo " Extra Repos: community, nisar, data-pipeline" } # Check and fix detached HEAD states in all repositories @@ -554,7 +630,7 @@ fix_all_detached_heads() { # Check all submodules echo "📁 Checking submodules..." - for sub in cloud comparison feed home localsite products projects realitystream swiper team; do + for sub in cloud comparison feed home localsite products projects realitystream swiper team trade codechat exiobase io profile reports community-forecasting; do if [ -d "$sub" ]; then echo "📁 Checking $sub..." cd "$sub" @@ -565,9 +641,9 @@ fix_all_detached_heads() { fi done - # Check trade repos - echo "📁 Checking trade repos..." - for repo in exiobase profile useeio.js io; do + # Check extra repos + echo "📁 Checking extra repos..." + for repo in community nisar data-pipeline; do if [ -d "$repo" ]; then echo "📁 Checking $repo..." cd "$repo" @@ -579,10 +655,10 @@ fix_all_detached_heads() { done if [ $fixed_count -gt 0 ]; then - echo "✅ Fixed detached HEAD states in $fixed_count repositories" - echo "💡 You may want to run './git.sh commit' to update submodule references" + echo "✅ All $fixed_count submodules pointed at main branch" + echo "💡 You may want to run './git.sh push' to update submodule references" else - echo "✅ No detached HEAD states found" + echo "✅ All submodules already on main branch" fi } @@ -608,7 +684,7 @@ update_all_remotes_for_user() { # Check all submodules echo "📁 Checking submodule remotes..." - for sub in cloud comparison feed home localsite products projects realitystream swiper team; do + for sub in cloud comparison feed home localsite products projects realitystream swiper team trade codechat exiobase io profile reports community-forecasting; do if [ -d "$sub" ]; then echo "📁 Checking $sub remotes..." cd "$sub" @@ -619,9 +695,9 @@ update_all_remotes_for_user() { fi done - # Check trade repos - echo "📁 Checking trade repo remotes..." - for repo in exiobase profile useeio.js io; do + # Check extra repos + echo "📁 Checking extra repo remotes..." + for repo in community nisar data-pipeline; do if [ -d "$repo" ]; then echo "📁 Checking $repo remotes..." cd "$repo" @@ -639,7 +715,73 @@ update_all_remotes_for_user() { fi } -# Create PR for webroot to its parent +# Check if GitHub Pages is enabled for a repository +check_github_pages() { + local user_login="$1" + local repo_name="$2" + + # Check if GitHub Pages is enabled using GitHub API + local pages_info=$(gh api "repos/$user_login/$repo_name/pages" 2>/dev/null || echo "") + if [ -n "$pages_info" ]; then + return 0 # Pages is enabled + else + return 1 # Pages is not enabled + fi +} + +# Enable GitHub Pages for a repository +enable_github_pages() { + local user_login="$1" + local repo_name="$2" + + echo "🌐 Enabling GitHub Pages for $user_login/$repo_name..." + + # Try to enable GitHub Pages using main branch + local result=$(gh api --method POST "repos/$user_login/$repo_name/pages" \ + -f source.branch=main \ + -f source.path=/ 2>/dev/null || echo "") + + if [ -n "$result" ]; then + echo "✅ GitHub Pages enabled for $user_login/$repo_name" + echo "📋 Site will be available at: https://$user_login.github.io/$repo_name" + return 0 + else + echo "⚠️ Could not enable GitHub Pages automatically" + echo "💡 Please manually enable GitHub Pages in your fork:" + echo " 1. Go to https://github.com/$user_login/$repo_name/settings/pages" + echo " 2. Set Source to 'Deploy from a branch'" + echo " 3. Select branch: main, folder: / (root)" + echo " 4. Click Save" + echo "📋 After setup, your site will be at: https://$user_login.github.io/$repo_name" + echo "" + echo "Options:" + echo " Y - Continue with PR creation (recommended)" + echo " N - Skip PR creation and continue with commit only" + echo " Q - Quit without creating PR or committing" + + read -p "Choose an option [Y/n/q]: " choice + case "${choice,,}" in + ""|y|yes) + echo "✅ Continuing with PR creation..." + return 0 + ;; + n|no) + echo "⚠️ Skipping PR creation as requested" + return 2 # Special return code for skip PR + ;; + q|quit) + echo "❌ Aborting commit and PR creation" + return 3 # Special return code for quit + ;; + *) + echo "Invalid choice. Defaulting to continue with PR..." + return 0 + ;; + esac + fi +} + +# Create PR for webroot to its parent with GitHub Pages integration create_webroot_pr() { local skip_pr="$1" @@ -677,71 +819,178 @@ create_webroot_pr() { return 1 fi + # Check and setup GitHub Pages for the fork + local pages_url="" + local pages_status="" + local pages_result=0 + + if check_github_pages "$user_login" "webroot"; then + pages_url="https://$user_login.github.io/webroot" + pages_status="✅ GitHub Pages is enabled" + echo "$pages_status: $pages_url" + else + echo "🔍 GitHub Pages not detected, attempting to enable..." + enable_github_pages "$user_login" "webroot" + pages_result=$? + + case $pages_result in + 0) + pages_url="https://$user_login.github.io/webroot" + pages_status="🌐 GitHub Pages enabled (may take a few minutes to be available)" + ;; + 2) + echo "🔄 Continuing with commit only (no PR as requested)" + return 0 # Skip PR creation but continue + ;; + 3) + echo "❌ Aborting PR creation as requested" + return 1 # Abort completely + ;; + *) + pages_url="https://$user_login.github.io/webroot" + pages_status="⚠️ GitHub Pages setup needed - continuing with PR creation" + ;; + esac + fi + + # Create enhanced PR body with review links + local pr_body="## Webroot Update + +Automated webroot update from git.sh commit workflow - includes submodule reference updates and configuration changes. + +## Review Links + +📋 **Live Preview**: [$pages_url]($pages_url) +🔗 **Fork Repository**: [https://github.com/$user_login/webroot](https://github.com/$user_login/webroot) + +## GitHub Pages Status +$pages_status + +--- +*Generated by git.sh commit workflow*" + local pr_url=$(gh pr create \ --title "Update webroot with submodule changes" \ - --body "Automated webroot update from git.sh commit workflow - includes submodule reference updates and configuration changes" \ + --body "$pr_body" \ --base main \ --head "$head_spec" \ --repo "$parent_account/webroot" 2>/dev/null || echo "") if [ -n "$pr_url" ]; then echo "🔄 Created webroot PR: $pr_url" + if [ -n "$pages_url" ]; then + echo "📋 Review at: $pages_url" + fi else echo "⚠️ Webroot PR creation failed or not needed" fi } -# Commit specific submodule -commit_submodule() { +# Push specific repository +push_specific_repo() { local name="$1" local skip_pr="$2" cd $(git rev-parse --show-toplevel) check_webroot - if [ -d "$name" ]; then - cd "$name" - commit_push "$name" "$skip_pr" - - # Update webroot submodule reference - cd .. - git submodule update --remote "$name" - if [ -n "$(git status --porcelain | grep $name)" ]; then - git add "$name" - git commit -m "Update $name submodule reference" - - # Try to push webroot changes - if git push 2>/dev/null; then - echo "✅ Updated $name submodule reference" - else - echo "🔄 Webroot push failed for $name - attempting PR workflow" - create_webroot_pr "$skip_pr" - fi - fi + # Auto-pull unless nopull/no pull is specified + if [[ "$skip_pr" != *"nopull"* ]] && [[ "$skip_pr" != *"no pull"* ]]; then + echo "🔄 Auto-pulling $name before push..." + pull_command "$name" + echo "✅ Pull completed for $name, proceeding with push..." + fi + + # Check if it's webroot + if [[ "$name" == "webroot" ]]; then + commit_push "webroot" "$skip_pr" - # Check if we need to create a webroot PR (for when webroot push succeeded but we want PR anyway) + # Check if webroot needs PR after direct changes local webroot_commits_ahead=$(git rev-list --count upstream/main..HEAD 2>/dev/null || echo "0") if [[ "$webroot_commits_ahead" -gt "0" ]] && [[ "$skip_pr" != "nopr" ]]; then create_webroot_pr "$skip_pr" fi - # Final push completion check echo "🔍 Checking for remaining unpushed commits..." final_push_completion_check - else - echo "⚠️ Repository not found: $name" + return + fi + + # Check if it's a submodule + if [[ " cloud comparison feed home localsite products projects realitystream swiper team trade codechat exiobase io profile reports community-forecasting " =~ " $name " ]]; then + if [ -d "$name" ]; then + cd "$name" + commit_push "$name" "$skip_pr" + + # Update webroot submodule reference + cd .. + git submodule update --remote "$name" + if [ -n "$(git status --porcelain | grep $name)" ]; then + git add "$name" + git commit -m "Update $name submodule reference" + + # Try to push webroot changes + if git push 2>/dev/null; then + echo "✅ Updated $name submodule reference" + else + echo "🔄 Webroot push failed for $name - attempting PR workflow" + create_webroot_pr "$skip_pr" + fi + fi + + # Check if we need to create a webroot PR (for when webroot push succeeded but we want PR anyway) + local webroot_commits_ahead=$(git rev-list --count upstream/main..HEAD 2>/dev/null || echo "0") + if [[ "$webroot_commits_ahead" -gt "0" ]] && [[ "$skip_pr" != "nopr" ]]; then + create_webroot_pr "$skip_pr" + fi + + # Final push completion check + echo "🔍 Checking for remaining unpushed commits..." + final_push_completion_check + else + echo "⚠️ Submodule not found: $name" + fi + return fi + + # Check if it's an extra repo + if [[ " community nisar data-pipeline " =~ " $name " ]]; then + if [ -d "$name" ]; then + cd "$name" + commit_push "$name" "$skip_pr" + cd .. + + echo "🔍 Checking for remaining unpushed commits..." + final_push_completion_check + else + echo "⚠️ Extra repo not found: $name" + fi + return + fi + + echo "⚠️ Repository not recognized: $name" + echo "Supported repositories:" + echo " Webroot: webroot" + echo " Submodules: cloud, comparison, feed, home, localsite, products, projects, realitystream, swiper, team, trade, codechat, exiobase, io, profile, reports, community-forecasting" + echo " Extra Repos: community, nisar, data-pipeline" } -# Commit all submodules -commit_submodules() { +# Push all submodules +push_submodules() { local skip_pr="$1" cd $(git rev-parse --show-toplevel) check_webroot - # Commit each submodule with changes - for sub in cloud comparison feed home localsite products projects realitystream swiper team; do + # Auto-pull unless nopull/no pull is specified + if [[ "$skip_pr" != *"nopull"* ]] && [[ "$skip_pr" != *"no pull"* ]]; then + echo "🔄 Auto-pulling submodules before push..." + pull_command + echo "✅ Pull completed for submodules, proceeding with push..." + fi + + # Push each submodule with changes + for sub in cloud comparison feed home localsite products projects realitystream swiper team trade codechat exiobase io profile reports community-forecasting; do [ ! -d "$sub" ] && continue cd "$sub" commit_push "$sub" "$skip_pr" @@ -762,14 +1011,21 @@ commit_submodules() { final_push_completion_check } -# Complete commit workflow -commit_all() { +# Complete push workflow +push_all() { local skip_pr="$1" cd $(git rev-parse --show-toplevel) check_webroot - # Commit webroot changes + # Auto-pull unless nopull/no pull is specified + if [[ "$skip_pr" != *"nopull"* ]] && [[ "$skip_pr" != *"no pull"* ]]; then + echo "🔄 Auto-pulling before push..." + pull_command + echo "✅ Pull completed, proceeding with push..." + fi + + # Push webroot changes commit_push "webroot" "$skip_pr" # Check if webroot needs PR after direct changes @@ -778,11 +1034,11 @@ commit_all() { create_webroot_pr "$skip_pr" fi - # Commit all submodules - commit_submodules "$skip_pr" + # Push all submodules + push_submodules "$skip_pr" - # Commit trade repos - for repo in exiobase profile useeio.js io; do + # Push extra repos + for repo in community nisar data-pipeline; do [ ! -d "$repo" ] && continue cd "$repo" commit_push "$repo" "$skip_pr" @@ -793,7 +1049,103 @@ commit_all() { echo "🔍 Checking for any remaining unpushed commits..." final_push_completion_check - echo "✅ Complete commit finished!" + echo "✅ Complete push finished!" + + # Check extra repos for uncommitted changes + check_extra_repos_for_changes +} + +# Check extra repos for uncommitted changes and prompt user +check_extra_repos_for_changes() { + cd $(git rev-parse --show-toplevel) + + local repos_with_changes=() + local repo_names=("community" "nisar" "data-pipeline") + + # Check each extra repo for changes + for repo in "${repo_names[@]}"; do + if [ -d "$repo" ]; then + cd "$repo" + if [ -n "$(git status --porcelain)" ]; then + repos_with_changes+=("$repo") + fi + cd .. + fi + done + + # If there are changes, prompt the user + if [ ${#repos_with_changes[@]} -gt 0 ]; then + echo "" + echo "📝 Extra repos with uncommitted changes detected:" + echo "" + + local i=1 + for repo in "${repos_with_changes[@]}"; do + echo " $i) $repo" + ((i++)) + done + echo " $i) all" + echo "" + + read -p "Which extra repo would you like to push? (1-$i or press Enter to skip): " choice + + if [ -n "$choice" ]; then + if [ "$choice" -eq "$i" ] 2>/dev/null; then + # Push all extra repos with changes + echo "🚀 Pushing all extra repos with changes..." + for repo in "${repos_with_changes[@]}"; do + push_extra_repo "$repo" + done + elif [ "$choice" -ge 1 ] && [ "$choice" -lt "$i" ] 2>/dev/null; then + # Push specific repo + local selected_repo="${repos_with_changes[$((choice-1))]}" + echo "🚀 Pushing $selected_repo..." + push_extra_repo "$selected_repo" + else + echo "❌ Invalid choice. Skipping extra repo push." + fi + else + echo "⏭️ Skipping extra repo push." + fi + fi +} + +# Push a specific extra repo +push_extra_repo() { + local repo_name="$1" + + cd $(git rev-parse --show-toplevel) + + if [ -d "$repo_name" ]; then + cd "$repo_name" + + if [ -n "$(git status --porcelain)" ]; then + git add . + git commit -m "Update $repo_name repository" + + if git push origin main 2>/dev/null; then + echo "✅ Successfully pushed $repo_name repository" + else + echo "⚠️ Push failed for $repo_name repository" + + # Try to create PR if it's a fork + local current_user=$(get_current_user) + if [ $? -eq 0 ] && [ -n "$current_user" ]; then + local remote_url=$(git remote get-url origin) + if [[ "$remote_url" =~ "$current_user/$repo_name" ]]; then + echo "🔄 Creating pull request for $repo_name..." + gh pr create --title "Update $repo_name" --body "Automated update from webroot integration" --base main --head main 2>/dev/null || echo "PR creation failed for $repo_name" + fi + fi + fi + else + echo "✅ No changes to push in $repo_name" + fi + + cd .. + else + echo "⚠️ Extra repo not found: $repo_name" + fi } # Check all repositories for unpushed commits and push them @@ -807,7 +1159,7 @@ final_push_completion_check() { fi # Check all submodules - for sub in cloud comparison feed home localsite products projects realitystream swiper team; do + for sub in cloud comparison feed home localsite products projects realitystream swiper team trade codechat exiobase io profile reports community-forecasting; do if [ -d "$sub" ]; then cd "$sub" if [ -n "$(git rev-list --count @{u}..HEAD 2>/dev/null)" ] && [ "$(git rev-list --count @{u}..HEAD 2>/dev/null)" != "0" ]; then @@ -818,8 +1170,8 @@ final_push_completion_check() { fi done - # Check trade repos - for repo in exiobase profile useeio.js io; do + # Check extra repos + for repo in community nisar data-pipeline; do if [ -d "$repo" ]; then cd "$repo" if [ -n "$(git rev-list --count @{u}..HEAD 2>/dev/null)" ] && [ "$(git rev-list --count @{u}..HEAD 2>/dev/null)" != "0" ]; then @@ -833,16 +1185,16 @@ final_push_completion_check() { # Main command dispatcher case "$1" in - "update") - update_command + "pull"|"pull-all") + pull_command "$2" ;; - "commit") + "push"|"push-all") if [ "$2" = "submodules" ]; then - commit_submodules "$3" + push_submodules "$3" + elif [ "$2" = "all" ] || [ -z "$2" ]; then + push_all "$2$3" # Handle both 'push' and 'push all [nopr]' elif [ -n "$2" ]; then - commit_submodule "$2" "$3" - else - commit_all "$2" + push_specific_repo "$2" "$3" fi ;; "fix-heads"|"fix") @@ -858,20 +1210,49 @@ case "$1" in update_all_remotes_for_user fi ;; + # Legacy command support with helpful messages + "update") + echo "⚠️ Please use 'pull' or 'pull all' instead of 'update'. Examples:" + echo " • pull - Pull all changes from webroot, submodules, and industry repos" + echo " • pull localsite - Pull changes for localsite submodule only" + echo " • pull webroot - Pull changes for webroot only" + echo "" + exit 1 + ;; + "commit") + echo "⚠️ Please use 'push' instead of 'commit'. Examples:" + echo " • push - Push all repositories with changes" + echo " • push localsite - Push changes for localsite submodule" + echo " • push webroot - Push changes for webroot only" + echo " • push all - Push all repositories with changes (same as 'push')" + echo "" + exit 1 + ;; *) - echo "Usage: ./git.sh [update|commit|fix|remotes|auth] [submodule_name|submodules] [nopr]" + echo "Usage: ./git.sh [pull|push|fix|remotes|auth] [repo_name|submodules|all] [nopr]" echo "" echo "Commands:" - echo " ./git.sh update - Run comprehensive update workflow" - echo " ./git.sh commit - Commit webroot, all submodules, and trade repos" - echo " ./git.sh commit [name] - Commit specific submodule" - echo " ./git.sh commit submodules - Commit all submodules only" + echo " ./git.sh pull - Pull all repositories (webroot + submodules + extra repos)" + echo " ./git.sh pull [repo_name] - Pull specific repository" + echo " ./git.sh push - Push all repositories with changes" + echo " ./git.sh push all - Push all repositories with changes (same as 'push')" + echo " ./git.sh push [repo_name] - Push specific repository" + echo " ./git.sh push submodules - Push all submodules only" echo " ./git.sh fix - Check and fix detached HEAD states in all repos" echo " ./git.sh remotes - Update all remotes to current GitHub user" echo " ./git.sh auth - Refresh git credentials for current GitHub user" echo "" + echo "Supported Repository Names:" + echo " Webroot: webroot" + echo " Submodules: cloud, comparison, feed, home, localsite, products, projects, realitystream, swiper, team, trade, codechat, exiobase, io, profile, reports, community-forecasting" + echo " Extra Repos: community, nisar, data-pipeline" + echo "" echo "Options:" echo " nopr - Skip PR creation on push failures" + echo "" + echo "Legacy Commands (deprecated):" + echo " update -> use 'pull' or 'pull all'" + echo " commit -> use 'push'" exit 1 ;; esac diff --git a/index.html b/index.html index 7209b2b0e..c4cda626f 100644 --- a/index.html +++ b/index.html @@ -6,13 +6,13 @@ Webroot - +