Ensure that profile and crawlconfig sorting of names and URLs is case-insensitive (#3016)

Fixes #3012 

- Add profile indexes 
- Add case-insensitive collation to indexes for crawlconfigs and profile
fields that should be sorted regardless of case (e.g. name, tags, URLs)
- Update tests

Author: tw4l

backend/btrixcloud/colls.py

@@ -12,7 +12,6 @@
 import asyncio
 import pymongo
 import aiohttp
-from pymongo.collation import Collation
 from fastapi import Depends, HTTPException, Response
 from fastapi.responses import StreamingResponse
 from starlette.requests import Request
@@ -55,6 +54,7 @@
     slug_from_name,
     get_duplicate_key_error_field,
     get_origin,
+    case_insensitive_collation,
 )
 
 if TYPE_CHECKING:
@@ -104,7 +104,6 @@ def set_page_ops(self, ops):
 
     async def init_index(self):
         """init lookup index"""
-        case_insensitive_collation = Collation(locale="en", strength=1)
         await self.collections.create_index(
             [("oid", pymongo.ASCENDING), ("name", pymongo.ASCENDING)],
             unique=True,
@@ -536,7 +535,7 @@ async def list_collections(
         )
 
         cursor = self.collections.aggregate(
-            aggregate, collation=pymongo.collation.Collation(locale="en")
+            aggregate, collation=case_insensitive_collation
         )
         results = await cursor.to_list(length=1)
         result = results[0]

backend/btrixcloud/crawlconfigs.py

@@ -72,6 +72,7 @@
     validate_language_code,
     is_url,
     browser_windows_from_scale,
+    case_insensitive_collation,
 )
 
 if TYPE_CHECKING:
@@ -207,7 +208,18 @@ async def init_index(self):
         )
 
         await self.crawl_configs.create_index(
-            [("oid", pymongo.ASCENDING), ("tags", pymongo.ASCENDING)]
+            [("oid", pymongo.ASCENDING), ("tags", pymongo.ASCENDING)],
+            collation=case_insensitive_collation,
+        )
+
+        await self.crawl_configs.create_index(
+            [("oid", pymongo.ASCENDING), ("name", pymongo.ASCENDING)],
+            collation=case_insensitive_collation,
+        )
+
+        await self.crawl_configs.create_index(
+            [("oid", pymongo.ASCENDING), ("firstSeed", pymongo.ASCENDING)],
+            collation=case_insensitive_collation,
         )
 
         await self.crawl_configs.create_index(
@@ -846,7 +858,9 @@ async def get_crawl_configs(
             ]
         )
 
-        cursor = self.crawl_configs.aggregate(aggregate)
+        cursor = self.crawl_configs.aggregate(
+            aggregate, collation=case_insensitive_collation
+        )
         results = await cursor.to_list(length=1)
         result = results[0]
         items = result["items"]

backend/btrixcloud/db.py

@@ -29,10 +29,13 @@
     from .background_jobs import BackgroundJobOps
     from .file_uploads import FileUploadOps
     from .crawlmanager import CrawlManager
+    from .profiles import ProfileOps
 else:
     UserManager = OrgOps = CrawlConfigOps = CrawlOps = CollectionOps = InviteOps = (
         StorageOps
-    ) = PageOps = BackgroundJobOps = FileUploadOps = CrawlLogOps = CrawlManager = object
+    ) = PageOps = BackgroundJobOps = FileUploadOps = CrawlLogOps = CrawlManager = (
+        ProfileOps
+    ) = object
 
 
 CURR_DB_VERSION = "0054"
@@ -103,6 +106,7 @@ async def update_and_prepare_db(
     background_job_ops: BackgroundJobOps,
     file_ops: FileUploadOps,
     crawl_log_ops: CrawlLogOps,
+    profile_ops: ProfileOps,
     crawl_manager: CrawlManager,
 ) -> None:
     """Prepare database for application.
@@ -139,6 +143,7 @@ async def update_and_prepare_db(
         storage_ops,
         file_ops,
         crawl_log_ops,
+        profile_ops,
     )
     await user_manager.create_super_user()
     await org_ops.create_default_org()
@@ -262,6 +267,7 @@ async def create_indexes(
     storage_ops,
     file_ops,
     crawl_log_ops,
+    profile_ops,
 ):
     """Create database indexes."""
     print("Creating database indexes", flush=True)
@@ -275,6 +281,7 @@ async def create_indexes(
     await storage_ops.init_index()
     await file_ops.init_index()
     await crawl_log_ops.init_index()
+    await profile_ops.init_index()
 
 
 # ============================================================================

backend/btrixcloud/main_migrations.py

@@ -29,7 +29,7 @@ async def main() -> int:
         _,
         page_ops,
         coll_ops,
-        _,
+        profile_ops,
         storage_ops,
         background_job_ops,
         _,
@@ -55,6 +55,7 @@ async def main() -> int:
         background_job_ops,
         file_ops,
         crawl_log_ops,
+        profile_ops,
         crawl_manager,
     )
 

backend/btrixcloud/migrations/migration_0039_coll_slugs.py

@@ -5,11 +5,10 @@
 from uuid import UUID
 
 from pymongo.errors import DuplicateKeyError
-from pymongo.collation import Collation
 import pymongo
 
 from btrixcloud.migrations import BaseMigration
-from btrixcloud.utils import slug_from_name
+from btrixcloud.utils import slug_from_name, case_insensitive_collation
 
 MIGRATION_VERSION = "0039"
 
@@ -53,7 +52,6 @@ async def migrate_up(self):
         Add slug to collections that don't have one yet, based on name
         """
         colls_mdb = self.mdb["collections"]
-        case_insensitive_collation = Collation(locale="en", strength=1)
 
         await colls_mdb.drop_indexes()
 

backend/btrixcloud/orgs.py

@@ -16,7 +16,6 @@
 
 from pydantic import ValidationError
 from pymongo import ReturnDocument
-from pymongo.collation import Collation
 from pymongo.errors import AutoReconnect, DuplicateKeyError
 
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -93,6 +92,7 @@
     validate_language_code,
     JSONSerializer,
     browser_windows_from_scale,
+    case_insensitive_collation,
 )
 
 if TYPE_CHECKING:
@@ -189,7 +189,6 @@ def set_default_primary_storage(self, storage: StorageRef):
 
     async def init_index(self) -> None:
         """init lookup index"""
-        case_insensitive_collation = Collation(locale="en", strength=1)
         while True:
             try:
                 await self.orgs.create_index(

backend/btrixcloud/profiles.py

@@ -20,7 +20,7 @@
 
 from fastapi import APIRouter, Depends, Request, HTTPException, Query
 from starlette.requests import Headers
-from pymongo import ReturnDocument
+import pymongo
 import aiohttp
 
 from .pagination import DEFAULT_PAGE_SIZE, paginated_format
@@ -48,7 +48,7 @@
     ListFilterType,
     ProfileSearchValuesResponse,
 )
-from .utils import dt_now, str_to_date
+from .utils import dt_now, str_to_date, case_insensitive_collation
 
 if TYPE_CHECKING:
     from .orgs import OrgOps
@@ -106,6 +106,26 @@ def set_crawlconfigs(self, crawlconfigs):
         """set crawlconfigs ops"""
         self.crawlconfigs = crawlconfigs
 
+    async def init_index(self):
+        """init lookup index"""
+        await self.profiles.create_index(
+            [("oid", pymongo.ASCENDING), ("name", pymongo.ASCENDING)],
+            collation=case_insensitive_collation,
+        )
+
+        await self.profiles.create_index(
+            [("oid", pymongo.ASCENDING), ("url", pymongo.ASCENDING)],
+            collation=case_insensitive_collation,
+        )
+
+        await self.profiles.create_index(
+            [("oid", pymongo.ASCENDING), ("created", pymongo.ASCENDING)]
+        )
+
+        await self.profiles.create_index(
+            [("oid", pymongo.ASCENDING), ("modified", pymongo.ASCENDING)]
+        )
+
     async def create_new_browser(
         self, org: Organization, user: User, profile_launch: ProfileLaunchBrowserIn
     ) -> BrowserId:
@@ -415,7 +435,7 @@ async def update_profile_from_crawl_upload(
                     "modifiedCrawlCid": cid,
                 }
             },
-            return_document=ReturnDocument.BEFORE,
+            return_document=pymongo.ReturnDocument.BEFORE,
         )
         if not res:
             return False
@@ -494,7 +514,9 @@ async def list_profiles(
             ]
         )
 
-        cursor = self.profiles.aggregate(aggregate)
+        cursor = self.profiles.aggregate(
+            aggregate, collation=case_insensitive_collation
+        )
         results = await cursor.to_list(length=1)
         result = results[0]
         items = result["items"]

backend/btrixcloud/utils.py

@@ -18,6 +18,7 @@
 from fastapi import HTTPException
 from fastapi.responses import StreamingResponse
 from iso639 import is_language
+from pymongo.collation import Collation
 from pymongo.errors import DuplicateKeyError
 from slugify import slugify
 
@@ -26,6 +27,8 @@
 
 browsers_per_pod = int(os.environ.get("NUM_BROWSERS", 1))
 
+case_insensitive_collation = Collation(locale="en", strength=1)
+
 
 class JSONSerializer(json.JSONEncoder):
     """Serializer class for json.dumps with UUID and datetime support"""

backend/test/conftest.py

@@ -207,7 +207,7 @@ def _crawler_create_config_only(crawler_auth_headers, default_org_id):
     # Start crawl.
     crawl_data = {
         "runNow": False,
-        "name": "Crawler User Test Crawl",
+        "name": "crawler User Test Crawl",
         "description": "crawler test crawl",
         "config": {
             "seeds": [{"url": "https://old.webrecorder.net/"}],
@@ -232,7 +232,7 @@ def crawler_crawl_id(crawler_auth_headers, default_org_id):
     # Start crawl.
     crawl_data = {
         "runNow": True,
-        "name": "Crawler User Test Crawl",
+        "name": "crawler User Test Crawl",
         "description": "crawler test crawl",
         "tags": ["wr-test-2"],
         "config": {"seeds": [{"url": "https://old.webrecorder.net/"}], "limit": 3},
@@ -702,7 +702,7 @@ def echo_server():
 PROFILE_DESC_UPDATED = "Updated profile used for backend tests"
 PROFILE_TAGS_UPDATED = ["profile", "profile-updated", "old-webrecorder"]
 
-PROFILE_2_NAME = "Second test profile"
+PROFILE_2_NAME = "second test profile"
 PROFILE_2_DESC = "Second profile used to test list endpoint"
 PROFILE_2_TAGS = ["profile", "specs-webrecorder"]
 

backend/test/test_crawl_config_search_values.py

@@ -44,7 +44,7 @@ def test_get_search_values_1(admin_auth_headers, default_org_id):
     )
     data = r.json()
     assert sorted(data["names"]) == sorted(
-        [NAME_1, "Admin Test Crawl", "Crawler User Test Crawl"]
+        [NAME_1, "Admin Test Crawl", "crawler User Test Crawl"]
     )
     assert sorted(data["descriptions"]) == sorted(
         ["Admin Test Crawl description", "crawler test crawl", DESCRIPTION_1]
@@ -74,7 +74,7 @@ def test_get_search_values_2(admin_auth_headers, default_org_id):
     )
     data = r.json()
     assert sorted(data["names"]) == sorted(
-        [NAME_1, NAME_2, "Admin Test Crawl", "Crawler User Test Crawl"]
+        [NAME_1, NAME_2, "Admin Test Crawl", "crawler User Test Crawl"]
     )
     assert sorted(data["descriptions"]) == sorted(
         [
@@ -111,7 +111,7 @@ def test_get_search_values_3(admin_auth_headers, default_org_id):
     )
     data = r.json()
     assert sorted(data["names"]) == sorted(
-        [NAME_1, NAME_2, "Admin Test Crawl", "Crawler User Test Crawl"]
+        [NAME_1, NAME_2, "Admin Test Crawl", "crawler User Test Crawl"]
     )
     assert sorted(data["descriptions"]) == sorted(
         [