merge mashb1 rq

Author: sunminghui

biz/queue/worker.py

@@ -8,7 +8,7 @@
 from biz.event.event_manager import event_manager
 from biz.gitlab.webhook_handler import filter_changes, MergeRequestHandler, PushHandler
 from biz.utils.code_reviewer import CodeReviewer
-from biz.utils.im import im_notifier
+from biz.utils.im import notifier
 from biz.utils.log import logger
 
 load_dotenv()
@@ -56,7 +56,7 @@ def handle_push_event(webhook_data: dict, gitlab_token: str, gitlab_url: str, gi
 
     except Exception as e:
         error_message = f'服务出现未知错误: {str(e)}\n{traceback.format_exc()}'
-        im_notifier.send_notification(content=error_message)
+        notifier.send_notification(content=error_message)
         logger.error('出现未知错误: %s', error_message)
 
 
@@ -119,5 +119,5 @@ def handle_merge_request_event(webhook_data: dict, gitlab_token: str, gitlab_url
 
     except Exception as e:
         error_message = f'AI Code Review 服务出现未知错误: {str(e)}\n{traceback.format_exc()}'
-        im_notifier.send_notification(content=error_message)
+        notifier.send_notification(content=error_message)
         logger.error('出现未知错误: %s', error_message)

biz/utils/code_reviewer.py

@@ -1,10 +1,12 @@
+import os
 import re
 import yaml
 import abc
 from typing import Dict, Any, List
 
 from biz.utils.log import logger
 from biz.llm.factory import Factory
+from biz.utils.token_util import count_tokens, truncate_text_by_tokens
 
 
 class BaseReviewer(abc.ABC):
@@ -54,16 +56,25 @@ def __init__(self):
         super().__init__("code_review_prompt")
 
     def review_and_strip_code(self, changes_text: str, commits_text: str = '') -> str:
-        # 如果超长，取前REVIEW_MAX_LENGTH字符
-        review_max_length = int(os.getenv('REVIEW_MAX_LENGTH', 5000))
+        """
+        Review判断changes_text超出取前REVIEW_MAX_TOKENS个token，超出则截断changes_text，
+        调用review_code方法，返回review_result，如果review_result是markdown格式，则去掉头尾的```
+        :param changes_text:
+        :param commits_text:
+        :return:
+        """
+        # 如果超长，取前REVIEW_MAX_TOKENS个token
+        review_max_tokens = int(os.getenv('REVIEW_MAX_TOKENS', 10000))
         # 如果changes为空,打印日志
         if not changes_text:
             logger.info('代码为空, diffs_text = %', str(changes_text))
             return '代码为空'
 
-        if len(changes_text) > review_max_length:
-            changes_text = changes_text[:review_max_length]
-            logger.info(f'文本超长，截段后content: {changes_text}')
+        # 计算tokens数量，如果超过REVIEW_MAX_TOKENS，截断changes_text
+        tokens_count = count_tokens(changes_text)
+        if tokens_count > review_max_tokens:
+            changes_text = truncate_text_by_tokens(changes_text, review_max_tokens)
+
         review_result = self.review_code(changes_text, commits_text).strip()
         if review_result.startswith("```markdown") and review_result.endswith("```"):
             return review_result[11:-3].strip()

biz/utils/queue.py

@@ -5,16 +5,21 @@
 from redis import Redis
 from rq import Queue
 
+from biz.utils.log import logger
+
 load_dotenv()
 queue_driver = os.getenv('QUEUE_DRIVER', 'async')
 
 if queue_driver == 'rq':
     queues = {}
 
+
 def handle_queue(function: callable, data: any, gitlab_token: str, gitlab_url: str, gitlab_url_slug: str):
     if queue_driver == 'rq':
         if gitlab_url_slug not in queues:
-            queues[gitlab_url_slug] = Queue(gitlab_url_slug, connection=Redis(os.getenv('REDIS_HOST', '127.0.0.1'), os.getenv('REDIS_PORT', 6379)))
+            logger.info(f'REDIS_HOST: {os.getenv("REDIS_HOST", "127.0.0.1")}，REDIS_PORT: {os.getenv("REDIS_PORT", 6379)}')
+            queues[gitlab_url_slug] = Queue(gitlab_url_slug, connection=Redis(os.getenv('REDIS_HOST', '127.0.0.1'),
+                                                                              os.getenv('REDIS_PORT', 6379)))
 
         queues[gitlab_url_slug].enqueue(function, data, gitlab_token, gitlab_url, gitlab_url_slug)
     else:

conf/supervisord.worker.conf

@@ -3,7 +3,8 @@ nodaemon=true
 user=root
 
 [program:worker]
-command=rq worker %(ENV_WORKER_QUEUE)s --url redis://redis:6379 --path /app
+#command=rq worker %(ENV_WORKER_QUEUE)s --url redis://redis:6379 --path /app
+command=rq worker gitlab_yj_edusoho_cn --url redis://redis:6379 --path /app
 autostart=true
 autorestart=true
 numprocs=1

docker-compose.yml

@@ -4,7 +4,7 @@ services:
       context: .
       dockerfile: Dockerfile
       target: dev
-    image: ghcr.io/sunmh207/ai-codereview-gitlab:latest
+    image: ghcr.io/sunmh207/ai-codereview-gitlab:1.3.1
     ports:
       - "5001:5001"
       - "5002:5002"
@@ -41,8 +41,10 @@ services:
 
   redis:
     image: redis:alpine
-#    ports:
-#      - "${REDIS_PORT}:6379"
+    env_file:
+      - ./conf/.env
+    ports:
+      - "6379:6379"
     volumes:
       - redis_data:/data
     restart: unless-stopped