🐛 fix: reduce concurrency due to strict rate limiter

Author: baoliay2008

app/crawler/contest_record_and_submission.py

@@ -42,7 +42,7 @@ async def request_contest_records(
     ]
     responses = await multi_http_request(
         {url: {"url": url, "method": "GET"} for url in url_list},
-        concurrent_num=20 if data_region == "US" else 50,
+        concurrent_num=5 if data_region == "US" else 10,
     )
     for res in responses:
         if res is None:

app/handler/user.py

@@ -104,8 +104,8 @@ async def update_all_users_in_database(
         await gather_with_limited_concurrency(
             [
                 # CN site has a strong rate limit
-                gather_with_limited_concurrency(cn_tasks, 4),
-                gather_with_limited_concurrency(us_tasks, 25),
+                gather_with_limited_concurrency(cn_tasks, 1),
+                gather_with_limited_concurrency(us_tasks, 5),
             ],
             30,
         )
@@ -163,6 +163,7 @@ async def save_users_of_contest(
         ]
     cursor = col.aggregate(pipeline)
     docs = await cursor.to_list(length=None)
+    logger.info(f"docs length = {len(docs)}")
     cn_tasks = []
     us_tasks = []
     for doc in docs:
@@ -181,8 +182,8 @@ async def save_users_of_contest(
     await gather_with_limited_concurrency(
         [
             # CN site has a strong rate limit
-            gather_with_limited_concurrency(cn_tasks, 4),
-            gather_with_limited_concurrency(us_tasks, 25),
-            30,
+            gather_with_limited_concurrency(cn_tasks, 1),
+            gather_with_limited_concurrency(us_tasks, 5),
         ],
+        30,
     )