From da561e88aaa577fcf48c819945b79e6717e55c5f Mon Sep 17 00:00:00 2001 From: emma Date: Tue, 4 Nov 2025 16:44:36 -0500 Subject: [PATCH 1/9] update orgs router to allow shared secret when determining org --- backend/btrixcloud/orgs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index db9b280465..39a837c033 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -1610,7 +1610,7 @@ async def org_public(oid: UUID): router = APIRouter( prefix="/orgs/{oid}", - dependencies=[Depends(org_dep)], + dependencies=[Depends(org_or_shared_secret_dep)], responses={404: {"description": "Not found"}}, ) From 91e6b9854976c8d5dc5ffdca1fa86ff52757ecfe Mon Sep 17 00:00:00 2001 From: emma Date: Tue, 2 Dec 2025 20:45:38 -0500 Subject: [PATCH 2/9] refactor quota updates to use aggregation pipeline replaces the previous multi-step update approach with a single aggregation pipeline that handles both "add" and "set" modes, ensuring atomic quota modifications --- backend/btrixcloud/orgs.py | 307 ++++++++++++++++++++++--------------- 1 file changed, 181 insertions(+), 126 deletions(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 39a837c033..0852d75d88 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -8,114 +8,112 @@ import math import os import time - -from uuid import UUID, uuid4 +from calendar import c from tempfile import NamedTemporaryFile - from typing import ( - Awaitable, - Optional, TYPE_CHECKING, - Dict, + Any, + AsyncGenerator, + Awaitable, Callable, + Dict, List, Literal, - AsyncGenerator, - Any, + Optional, ) +from uuid import UUID, uuid4 +import json_stream +from aiostream import stream +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import StreamingResponse from motor.motor_asyncio import AsyncIOMotorDatabase from pydantic import ValidationError from pymongo import ReturnDocument from pymongo.errors import AutoReconnect, DuplicateKeyError -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import StreamingResponse -import json_stream -from aiostream import stream - from .models import ( - SUCCESSFUL_STATES, + ACTIVE, + MAX_BROWSER_WINDOWS, + MAX_CRAWL_SCALE, + PAUSED_PAYMENT_FAILED, + REASON_PAUSED, RUNNING_STATES, + SUCCESSFUL_STATES, WAITING_STATES, + AddedResponse, + AddedResponseId, + AddToOrgRequest, BaseCrawl, + Collection, + ConfigRevision, + Crawl, + CrawlConfig, + CrawlConfigDefaults, + DeleteCrawlList, + DeletedResponseId, + InvitePending, + InviteToOrgRequest, + OrgAcceptInviteResponse, Organization, - PlansResponse, - StorageRef, + OrgCreate, + OrgDeleteInviteResponse, + OrgImportResponse, + OrgInviteResponse, + OrgMetrics, + OrgOut, + OrgOutExport, + OrgProxies, + OrgPublicProfileUpdate, OrgQuotas, OrgQuotasIn, OrgQuotaUpdate, - OrgReadOnlyUpdate, OrgReadOnlyOnCancel, - OrgMetrics, + OrgReadOnlyUpdate, + OrgSlugsResponse, OrgWebhookUrls, - OrgCreate, - OrgProxies, - Subscription, - SubscriptionUpdate, - SubscriptionCancel, - RenameOrg, - UpdateRole, - RemovePendingInvite, - RemoveFromOrg, - AddToOrgRequest, - InvitePending, - InviteToOrgRequest, - UserRole, - User, + PageWithAllQA, PaginatedInvitePendingResponse, PaginatedOrgOutResponse, - CrawlConfig, - Crawl, - CrawlConfigDefaults, - UploadedCrawl, - ConfigRevision, + PlansResponse, Profile, - Collection, - OrgOut, - OrgOutExport, - PageWithAllQA, - DeleteCrawlList, - PAUSED_PAYMENT_FAILED, - REASON_PAUSED, - ACTIVE, - DeletedResponseId, - UpdatedResponse, - AddedResponse, - AddedResponseId, - SuccessResponseId, - OrgInviteResponse, - OrgAcceptInviteResponse, - OrgDeleteInviteResponse, RemovedResponse, - OrgSlugsResponse, - OrgImportResponse, - OrgPublicProfileUpdate, - MAX_BROWSER_WINDOWS, - MAX_CRAWL_SCALE, + RemoveFromOrg, + RemovePendingInvite, + RenameOrg, + StorageRef, + Subscription, + SubscriptionCancel, + SubscriptionUpdate, + SuccessResponseId, + UpdatedResponse, + UpdateRole, + UploadedCrawl, + User, + UserRole, ) from .pagination import DEFAULT_PAGE_SIZE, paginated_format from .utils import ( - dt_now, - slug_from_name, - validate_slug, - get_duplicate_key_error_field, - validate_language_code, JSONSerializer, browser_windows_from_scale, case_insensitive_collation, + dt_now, + get_duplicate_key_error_field, + slug_from_name, + validate_language_code, + validate_slug, ) if TYPE_CHECKING: - from .invites import InviteOps + from .background_jobs import BackgroundJobOps from .basecrawls import BaseCrawlOps from .colls import CollectionOps + from .crawlmanager import CrawlManager + from .file_uploads import FileUploadOps + from .invites import InviteOps + from .pages import PageOps from .profiles import ProfileOps from .users import UserManager - from .background_jobs import BackgroundJobOps - from .pages import PageOps - from .file_uploads import FileUploadOps - from .crawlmanager import CrawlManager else: InviteOps = BaseCrawlOps = ProfileOps = CollectionOps = object BackgroundJobOps = UserManager = PageOps = FileUploadOps = CrawlManager = object @@ -627,74 +625,131 @@ async def update_quotas( quotas.context = None - previous_extra_mins = ( - org.quotas.extraExecMinutes - if (org.quotas and org.quotas.extraExecMinutes) - else 0 - ) - previous_gifted_mins = ( - org.quotas.giftedExecMinutes - if (org.quotas and org.quotas.giftedExecMinutes) - else 0 - ) - - if mode == "add": - increment_update: dict[str, Any] = { - "$inc": {}, + update: list[dict[str, Any]] = [ + { + "$set": { + "quotaUpdates": { + "$concatArrays": [ + "$quotaUpdates", + [{"modified": dt_now(), "update": {}}], + ] + }, + } } + ] - for field, value in quotas.model_dump( - exclude_unset=True, exclude_defaults=True, exclude_none=True - ).items(): - if field == "context" or value is None: - continue - inc = max(value, -org.quotas.model_dump().get(field, 0)) - increment_update["$inc"][f"quotas.{field}"] = inc + computed_quotas = {} - updated_org = await self.orgs.find_one_and_update( - {"_id": org.id}, - increment_update, - projection={"quotas": True}, - return_document=ReturnDocument.AFTER, + if mode == "add": + update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0]["context"] = ( + context ) - quotas = OrgQuotasIn(**updated_org["quotas"]) - - update: dict[str, dict[str, dict[str, Any] | int]] = { - "$push": { - "quotaUpdates": OrgQuotaUpdate( - modified=dt_now(), - update=OrgQuotas( - **quotas.model_dump( - exclude_unset=True, exclude_defaults=True, exclude_none=True - ) - ), - context=context, - ).model_dump() - }, - "$inc": {}, - "$set": {}, - } - - if mode == "set": + for field, value in quotas.model_dump().items(): + if field == "context": + continue + if value is None: + # set value of field in pushed update to current value in `quotas` + update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0]["update"][ + field + ] = f"$quotas.{field}" + computed_quotas[field] = f"$quotas.{field}" + continue + new_value = { + "$add": [ + { + "$cond": { + "if": { + "$gt": [ + {"$multiply": [f"$quotas.{field}", -1]}, + value, + ] + }, + "then": {"$multiply": [f"$quotas.{field}", -1]}, + "else": value, + } + }, + f"$quotas.{field}", + ] + } + # set value of field in pushed update to current value in quotas + increment + update[0]["$set"][f"quotas.{field}"] = new_value + update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0]["update"][ + field + ] = new_value + computed_quotas[field] = new_value + + elif mode == "set": increment_update = quotas.model_dump( exclude_unset=True, exclude_defaults=True, exclude_none=True ) - update["$set"]["quotas"] = increment_update + for field, value in increment_update.items(): + update[0]["$set"][f"quotas.{field}"] = value + computed_quotas[field] = value + update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0] = OrgQuotaUpdate( + modified=dt_now(), + update=OrgQuotas( + **quotas.model_dump( + exclude_unset=True, exclude_defaults=True, exclude_none=True + ) + ), + context=context, + ).model_dump() # Inc org available fields for extra/gifted execution time as needed - if quotas.extraExecMinutes is not None: - extra_secs_diff = (quotas.extraExecMinutes - previous_extra_mins) * 60 - if org.extraExecSecondsAvailable + extra_secs_diff <= 0: - update["$set"]["extraExecSecondsAvailable"] = 0 - else: - update["$inc"]["extraExecSecondsAvailable"] = extra_secs_diff + for extra_or_gifted in ["extra", "gifted"]: + previous_mins = { + "$cond": { + "if": { + "$or": [ + {"$ne": [f"$quotas.{extra_or_gifted}ExecMinutes", 0]}, + {"$ne": [f"$quotas.{extra_or_gifted}ExecMinutes", None]}, + ] + }, + "then": f"$quotas.{extra_or_gifted}ExecMinutes", + "else": 0, + } + } - if quotas.giftedExecMinutes is not None: - gifted_secs_diff = (quotas.giftedExecMinutes - previous_gifted_mins) * 60 - if org.giftedExecSecondsAvailable + gifted_secs_diff <= 0: - update["$set"]["giftedExecSecondsAvailable"] = 0 - else: - update["$inc"]["giftedExecSecondsAvailable"] = gifted_secs_diff + secs_diff = { + "$multiply": [ + { + "$subtract": [ + computed_quotas[f"{extra_or_gifted}ExecMinutes"], + previous_mins, + ] + }, + 60, + ] + } + + update[0]["$set"][f"{extra_or_gifted}ExecSecondsAvailable"] = { + "$cond": { + "if": {"$ne": [f"${extra_or_gifted}ExecSecondsAvailable", None]}, + "then": { + "$cond": { + "if": { + "$lte": [ + { + "$add": [ + f"${extra_or_gifted}ExecSecondsAvailable", + secs_diff, + ] + }, + 0, + ] + }, + "then": 0, + "else": { + "$add": [ + f"${extra_or_gifted}ExecSecondsAvailable", + secs_diff, + ] + }, + } + }, + "else": f"${extra_or_gifted}ExecSecondsAvailable", + } + } await self.orgs.find_one_and_update({"_id": org.id}, update) From 1fdc072ff80d46f11b997d4ca3a9d285802790f7 Mon Sep 17 00:00:00 2001 From: emma Date: Tue, 18 Nov 2025 19:09:45 -0500 Subject: [PATCH 3/9] format --- backend/btrixcloud/orgs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 0852d75d88..a3e1cbd0bb 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -641,9 +641,9 @@ async def update_quotas( computed_quotas = {} if mode == "add": - update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0]["context"] = ( - context - ) + update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0][ + "context" + ] = context for field, value in quotas.model_dump().items(): if field == "context": continue From f8290de016568cecc2ab80e21e8ae722332876ad Mon Sep 17 00:00:00 2001 From: emma Date: Tue, 2 Dec 2025 21:01:01 -0500 Subject: [PATCH 4/9] fix accidentally-changed org router dep --- backend/btrixcloud/orgs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index a3e1cbd0bb..7efcce5b7a 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -641,9 +641,9 @@ async def update_quotas( computed_quotas = {} if mode == "add": - update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0][ - "context" - ] = context + update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0]["context"] = ( + context + ) for field, value in quotas.model_dump().items(): if field == "context": continue @@ -1665,7 +1665,7 @@ async def org_public(oid: UUID): router = APIRouter( prefix="/orgs/{oid}", - dependencies=[Depends(org_or_shared_secret_dep)], + dependencies=[Depends(org_dep)], responses={404: {"description": "Not found"}}, ) From 4b374dfef67a24768f3eafbd4561588c8ea8f762 Mon Sep 17 00:00:00 2001 From: emma Date: Tue, 2 Dec 2025 21:25:51 -0500 Subject: [PATCH 5/9] format --- backend/btrixcloud/orgs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 7efcce5b7a..81becd9860 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -641,9 +641,9 @@ async def update_quotas( computed_quotas = {} if mode == "add": - update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0]["context"] = ( - context - ) + update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0][ + "context" + ] = context for field, value in quotas.model_dump().items(): if field == "context": continue From 6340e91ee0b7b996b3f97bf2ef4ec72c7e108faa Mon Sep 17 00:00:00 2001 From: emma Date: Tue, 2 Dec 2025 21:36:16 -0500 Subject: [PATCH 6/9] undo accidentally-reformatted imports --- backend/btrixcloud/orgs.py | 136 +++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 67 deletions(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 81becd9860..52ebc0676e 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -8,112 +8,114 @@ import math import os import time -from calendar import c + +from uuid import UUID, uuid4 from tempfile import NamedTemporaryFile + from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, Awaitable, - Callable, - Dict, - List, Literal, Optional, + TYPE_CHECKING, + Dict, + Callable, + List, + AsyncGenerator, + Any, ) -from uuid import UUID, uuid4 -import json_stream -from aiostream import stream -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import StreamingResponse from motor.motor_asyncio import AsyncIOMotorDatabase from pydantic import ValidationError from pymongo import ReturnDocument from pymongo.errors import AutoReconnect, DuplicateKeyError +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import StreamingResponse +import json_stream +from aiostream import stream + from .models import ( - ACTIVE, - MAX_BROWSER_WINDOWS, - MAX_CRAWL_SCALE, - PAUSED_PAYMENT_FAILED, - REASON_PAUSED, - RUNNING_STATES, SUCCESSFUL_STATES, + RUNNING_STATES, WAITING_STATES, - AddedResponse, - AddedResponseId, - AddToOrgRequest, BaseCrawl, - Collection, - ConfigRevision, - Crawl, - CrawlConfig, - CrawlConfigDefaults, - DeleteCrawlList, - DeletedResponseId, - InvitePending, - InviteToOrgRequest, - OrgAcceptInviteResponse, Organization, - OrgCreate, - OrgDeleteInviteResponse, - OrgImportResponse, - OrgInviteResponse, - OrgMetrics, - OrgOut, - OrgOutExport, - OrgProxies, - OrgPublicProfileUpdate, + PlansResponse, + StorageRef, OrgQuotas, OrgQuotasIn, OrgQuotaUpdate, - OrgReadOnlyOnCancel, OrgReadOnlyUpdate, - OrgSlugsResponse, + OrgReadOnlyOnCancel, + OrgMetrics, OrgWebhookUrls, - PageWithAllQA, - PaginatedInvitePendingResponse, - PaginatedOrgOutResponse, - PlansResponse, - Profile, - RemovedResponse, - RemoveFromOrg, - RemovePendingInvite, - RenameOrg, - StorageRef, + OrgCreate, + OrgProxies, Subscription, - SubscriptionCancel, SubscriptionUpdate, - SuccessResponseId, - UpdatedResponse, + SubscriptionCancel, + RenameOrg, UpdateRole, - UploadedCrawl, - User, + RemovePendingInvite, + RemoveFromOrg, + AddToOrgRequest, + InvitePending, + InviteToOrgRequest, UserRole, + User, + PaginatedInvitePendingResponse, + PaginatedOrgOutResponse, + CrawlConfig, + Crawl, + CrawlConfigDefaults, + UploadedCrawl, + ConfigRevision, + Profile, + Collection, + OrgOut, + OrgOutExport, + PageWithAllQA, + DeleteCrawlList, + PAUSED_PAYMENT_FAILED, + REASON_PAUSED, + ACTIVE, + DeletedResponseId, + UpdatedResponse, + AddedResponse, + AddedResponseId, + SuccessResponseId, + OrgInviteResponse, + OrgAcceptInviteResponse, + OrgDeleteInviteResponse, + RemovedResponse, + OrgSlugsResponse, + OrgImportResponse, + OrgPublicProfileUpdate, + MAX_BROWSER_WINDOWS, + MAX_CRAWL_SCALE, ) from .pagination import DEFAULT_PAGE_SIZE, paginated_format from .utils import ( - JSONSerializer, - browser_windows_from_scale, - case_insensitive_collation, dt_now, - get_duplicate_key_error_field, slug_from_name, - validate_language_code, validate_slug, + get_duplicate_key_error_field, + validate_language_code, + JSONSerializer, + browser_windows_from_scale, + case_insensitive_collation, ) if TYPE_CHECKING: - from .background_jobs import BackgroundJobOps + from .invites import InviteOps from .basecrawls import BaseCrawlOps from .colls import CollectionOps - from .crawlmanager import CrawlManager - from .file_uploads import FileUploadOps - from .invites import InviteOps - from .pages import PageOps from .profiles import ProfileOps from .users import UserManager + from .background_jobs import BackgroundJobOps + from .pages import PageOps + from .file_uploads import FileUploadOps + from .crawlmanager import CrawlManager else: InviteOps = BaseCrawlOps = ProfileOps = CollectionOps = object BackgroundJobOps = UserManager = PageOps = FileUploadOps = CrawlManager = object From b1bbbaecc40db6a4332a865031d1e23e362152ba Mon Sep 17 00:00:00 2001 From: emma Date: Tue, 2 Dec 2025 21:36:27 -0500 Subject: [PATCH 7/9] fix mypy type issue --- backend/btrixcloud/orgs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 52ebc0676e..a90ca5ba28 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -640,7 +640,7 @@ async def update_quotas( } ] - computed_quotas = {} + computed_quotas: dict[str, Any] = {} if mode == "add": update[0]["$set"]["quotaUpdates"]["$concatArrays"][1][0][ From 10bbb2630dc2a703aa28694efa26d4461703524e Mon Sep 17 00:00:00 2001 From: emma Date: Wed, 3 Dec 2025 15:22:52 -0500 Subject: [PATCH 8/9] skip updates if no computed quota is present --- backend/btrixcloud/orgs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index a90ca5ba28..95347dba60 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -699,6 +699,8 @@ async def update_quotas( # Inc org available fields for extra/gifted execution time as needed for extra_or_gifted in ["extra", "gifted"]: + if not computed_quotas[f"{extra_or_gifted}ExecMinutes"]: + continue previous_mins = { "$cond": { "if": { From d68bdfd5f65cea1f2535c244253f213333768a1d Mon Sep 17 00:00:00 2001 From: emma Date: Wed, 3 Dec 2025 16:12:37 -0500 Subject: [PATCH 9/9] fix skip check --- backend/btrixcloud/orgs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 95347dba60..1132acd0d8 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -699,7 +699,7 @@ async def update_quotas( # Inc org available fields for extra/gifted execution time as needed for extra_or_gifted in ["extra", "gifted"]: - if not computed_quotas[f"{extra_or_gifted}ExecMinutes"]: + if f"{extra_or_gifted}ExecMinutes" not in computed_quotas: continue previous_mins = { "$cond": {