Merge branch 'main' into feat/add-feed-comment-count

Author: CodeVac513

.github/workflows/deploy_feed-crawler.yml

@@ -40,6 +40,6 @@ jobs:
             echo "AI_RATE_LIMIT_COUNT=${{ vars.AI_RATE_LIMIT_COUNT }}" >> env/.env.prod
 
             cd /var/web05-Denamu
-            docker-compose -f docker-compose/docker-compose.prod.yml up --build --no-deps -d feed-crawler
+            docker-compose -f docker-compose/docker-compose.prod.yml up --build --no-deps --force-recreate -d feed-crawler
             docker image prune -f
             docker builder prune -f

.github/workflows/deploy_server.yml

@@ -49,6 +49,6 @@ jobs:
             echo "GITHUB_CLIENT_SECRET=${{secrets.GIT_CLIENT_SECRET}}" >> env/.env.prod
 
             cd /var/web05-Denamu
-            docker-compose -f docker-compose/docker-compose.prod.yml up --build --no-deps -d app
+            docker-compose -f docker-compose/docker-compose.prod.yml up --build --no-deps --force-recreate -d app
             docker image prune -f
             docker builder prune -f

.gitignore

@@ -33,6 +33,9 @@ certificate.crt
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
+### static files
+**/objects
+
 # Local History for Visual Studio Code
 .history/
 

docker-compose/docker-compose.prod.yml

@@ -21,6 +21,7 @@ services:
         condition: service_healthy
     volumes:
       - ../server/logs:/var/web05-Denamu/server/logs
+      - /var/web05-Denamu/objects:/var/web05-Denamu/objects
     environment:
       NODE_ENV: "PROD"
       TZ: "Asia/Seoul"

feed-crawler/src/common/parser/base-feed-parser.ts

@@ -0,0 +1,87 @@
+import { XMLParser } from 'fast-xml-parser';
+import { FeedDetail, RssObj } from '../types';
+import {
+  FEED_AI_SUMMARY_IN_PROGRESS_MESSAGE,
+  ONE_MINUTE,
+  TIME_INTERVAL,
+} from '../constant';
+import { ParserUtil } from './utils/parser-util';
+
+export interface RawFeed {
+  title: string;
+  link: string;
+  pubDate: string;
+  description: string;
+}
+
+export abstract class BaseFeedParser {
+  protected readonly xmlParser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: '@_',
+    parseAttributeValue: true,
+    trimValues: true,
+  });
+  protected readonly parserUtil: ParserUtil;
+
+  constructor(parserUtil: ParserUtil) {
+    this.parserUtil = parserUtil;
+  }
+
+  async parseFeed(rssObj: RssObj, xmlData: string): Promise<FeedDetail[]> {
+    // 각 포맷(atom1.0, rss2.0 등...)
+    const rawFeeds = this.extractRawFeeds(xmlData);
+    const timeMatchedFeeds = this.filterByTime(rawFeeds);
+    const detailedFeeds = await this.convertToFeedDetails(
+      rssObj,
+      timeMatchedFeeds,
+    );
+
+    return detailedFeeds;
+  }
+
+  abstract canParse(xmlData: string): boolean;
+  protected abstract extractRawFeeds(xmlData: string): RawFeed[];
+
+  private filterByTime(rawFeeds: RawFeed[]): RawFeed[] {
+    const now = new Date().setSeconds(0, 0);
+    return rawFeeds.filter((item) => {
+      const pubDate = new Date(item.pubDate).setSeconds(0, 0);
+      const timeDiff = (now - pubDate) / (ONE_MINUTE * TIME_INTERVAL);
+      return timeDiff >= 0 && timeDiff < 1;
+    });
+  }
+
+  private async convertToFeedDetails(
+    rssObj: RssObj,
+    rawFeeds: RawFeed[],
+  ): Promise<FeedDetail[]> {
+    return Promise.all(
+      rawFeeds.map(async (feed) => {
+        const imageUrl = await this.parserUtil.getThumbnailUrl(feed.link);
+        const date = new Date(feed.pubDate);
+        const formattedDate = date.toISOString().slice(0, 19).replace('T', ' ');
+
+        const content = (feed.description || '')
+          .replace(/<[^>]*>/g, '')
+          .replace(/&nbsp;|&#160;/g, ' ')
+          .replace(/&[^;]+;/g, '')
+          .replace(/\s+/g, ' ')
+          .trim();
+
+        return {
+          id: null,
+          blogId: rssObj.id,
+          blogName: rssObj.blogName,
+          blogPlatform: rssObj.blogPlatform,
+          pubDate: formattedDate,
+          title: feed.title,
+          link: decodeURIComponent(feed.link),
+          imageUrl: imageUrl,
+          content: content,
+          summary: FEED_AI_SUMMARY_IN_PROGRESS_MESSAGE,
+          deathCount: 0,
+        } as FeedDetail;
+      }),
+    );
+  }
+}

feed-crawler/src/common/parser/feed-parser-manager.ts

@@ -0,0 +1,56 @@
+import { inject, injectable } from 'tsyringe';
+import { FeedDetail, RssObj } from '../types';
+import { Rss20Parser } from './formats/rss20-parser';
+import { Atom10Parser } from './formats/atom10-parser';
+import { BaseFeedParser } from './base-feed-parser';
+import { DEPENDENCY_SYMBOLS } from '../../types/dependency-symbols';
+import logger from '../logger';
+
+@injectable()
+export class FeedParserManager {
+  private readonly parsers: BaseFeedParser[];
+
+  constructor(
+    @inject(DEPENDENCY_SYMBOLS.Rss20Parser) rss20Parser: Rss20Parser,
+    @inject(DEPENDENCY_SYMBOLS.Atom10Parser) atom10Parser: Atom10Parser,
+  ) {
+    this.parsers = [rss20Parser, atom10Parser];
+  }
+
+  async fetchAndParse(rssObj: RssObj): Promise<FeedDetail[]> {
+    try {
+      const response = await fetch(rssObj.rssUrl, {
+        headers: {
+          Accept:
+            'application/rss+xml, application/xml, text/xml, application/atom+xml',
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(`${rssObj.rssUrl}에서 피드 데이터 가져오기 실패`);
+      }
+
+      const xmlData = await response.text();
+
+      const parser = this.findSuitableParser(xmlData);
+      if (!parser) {
+        throw new Error(`지원하지 않는 피드 형식: ${rssObj.rssUrl} / `);
+      }
+      logger.info(`${rssObj.blogName}: ${parser.constructor.name} 사용`);
+
+      return await parser.parseFeed(rssObj, xmlData);
+    } catch (error) {
+      logger.warn(`[${rssObj.rssUrl}] 피드 파싱 중 오류 발생: ${error}`);
+      return [];
+    }
+  }
+
+  private findSuitableParser(xmlData: string): BaseFeedParser | null {
+    for (const parser of this.parsers) {
+      if (parser.canParse(xmlData)) {
+        return parser;
+      }
+    }
+    return null;
+  }
+}

feed-crawler/src/common/parser/formats/atom10-parser.ts

@@ -0,0 +1,50 @@
+import { inject, injectable } from 'tsyringe';
+import { BaseFeedParser, RawFeed } from '../base-feed-parser';
+import { ParserUtil } from '../utils/parser-util';
+import { DEPENDENCY_SYMBOLS } from '../../../types/dependency-symbols';
+
+@injectable()
+export class Atom10Parser extends BaseFeedParser {
+  constructor(@inject(DEPENDENCY_SYMBOLS.ParserUtil) parserUtil: ParserUtil) {
+    super(parserUtil);
+  }
+  canParse(xmlData: string): boolean {
+    try {
+      const parsed = this.xmlParser.parse(xmlData);
+      return !!parsed.feed?.entry;
+    } catch {
+      return false;
+    }
+  }
+
+  protected extractRawFeeds(xmlData: string): RawFeed[] {
+    const parsed = this.xmlParser.parse(xmlData);
+
+    let entries = parsed.feed.entry;
+    if (!Array.isArray(entries)) {
+      entries = [entries];
+    }
+
+    return entries.map((entry: any) => ({
+      title: this.parserUtil.customUnescape(entry.title),
+      link: this.extractLink(entry.link),
+      pubDate: entry.published || entry.updated,
+      description: entry.summary || entry.content || '',
+    }));
+  }
+
+  private extractLink(linkData: any): string {
+    // link 태그가 속성없이 문자 형태 그대로일 경우
+    if (typeof linkData === 'string') {
+      return linkData;
+    }
+
+    // link 태그가 여러개인 경우
+    if (Array.isArray(linkData)) {
+      const alternateLink = linkData.find((l) => l['@_rel'] === 'alternate');
+      return alternateLink['@_href'] || '';
+    }
+
+    return linkData['@_href'] || linkData?.href || '';
+  }
+}

feed-crawler/src/common/parser/formats/rss20-parser.ts

@@ -0,0 +1,34 @@
+import { inject, injectable } from 'tsyringe';
+import { BaseFeedParser, RawFeed } from '../base-feed-parser';
+import { ParserUtil } from '../utils/parser-util';
+import { DEPENDENCY_SYMBOLS } from '../../../types/dependency-symbols';
+
+@injectable()
+export class Rss20Parser extends BaseFeedParser {
+  constructor(@inject(DEPENDENCY_SYMBOLS.ParserUtil) parserUtil: ParserUtil) {
+    super(parserUtil);
+  }
+  canParse(xmlData: string): boolean {
+    try {
+      const parsed = this.xmlParser.parse(xmlData);
+      return !!parsed.rss?.channel?.item;
+    } catch {
+      return false;
+    }
+  }
+
+  protected extractRawFeeds(xmlData: string): RawFeed[] {
+    const parsed = this.xmlParser.parse(xmlData);
+
+    if (!Array.isArray(parsed.rss.channel.item)) {
+      parsed.rss.channel.item = [parsed.rss.channel.item];
+    }
+
+    return parsed.rss.channel.item.map((feed: any) => ({
+      title: this.parserUtil.customUnescape(feed.title),
+      link: feed.link,
+      pubDate: feed.pubDate,
+      description: feed.description,
+    }));
+  }
+}

feed-crawler/src/common/rss-parser.ts …r/src/common/parser/utils/parser-util.tsfeed-crawler/src/common/rss-parser.ts renamed to feed-crawler/src/common/parser/utils/parser-util.ts feed-crawler/src/common/rss-parser.ts renamed to feed-crawler/src/common/parser/utils/parser-util.ts

@@ -1,12 +1,14 @@
-import logger from "./logger";
-import { parse } from "node-html-parser";
-import { unescape } from "html-escaper";
+import { injectable } from 'tsyringe';
+import logger from '../../logger';
+import { parse } from 'node-html-parser';
+import { unescape } from 'html-escaper';
 
-export class RssParser {
+@injectable()
+export class ParserUtil {
   async getThumbnailUrl(feedUrl: string) {
     const response = await fetch(feedUrl, {
       headers: {
-        Accept: "text/html",
+        Accept: 'text/html',
       },
     });
     if (!response.ok) {
@@ -16,9 +18,9 @@ export class RssParser {
     const htmlData = await response.text();
     const htmlRootElement = parse(htmlData);
     const metaImage = htmlRootElement.querySelector(
-      'meta[property="og:image"]'
+      'meta[property="og:image"]',
     );
-    let thumbnailUrl = metaImage?.getAttribute("content") ?? "";
+    let thumbnailUrl = metaImage?.getAttribute('content') ?? '';
 
     if (!thumbnailUrl.length) {
       logger.warn(`${feedUrl}에서 썸네일 추출 실패`);
@@ -42,12 +44,12 @@ export class RssParser {
 
   customUnescape(feedTitle: string): string {
     const escapeEntity = {
-      "·": "·",
-      " ": " ",
+      '·': '·',
+      ' ': ' ',
     };
     Object.keys(escapeEntity).forEach((escapeKey) => {
       const value = escapeEntity[escapeKey];
-      const regex = new RegExp(escapeKey, "g");
+      const regex = new RegExp(escapeKey, 'g');
       feedTitle = feedTitle.replace(regex, value);
     });
 

feed-crawler/src/container.ts

@@ -6,8 +6,12 @@ import { RssRepository } from './repository/rss.repository';
 import { FeedRepository } from './repository/feed.repository';
 import { RedisConnection } from './common/redis-access';
 import { TagMapRepository } from './repository/tag-map.repository';
-import { RssParser } from './common/rss-parser';
+import { ParserUtil } from './common/parser/utils/parser-util';
 import { ClaudeService } from './claude.service';
+import { FeedParserManager } from './common/parser/feed-parser-manager';
+import { Rss20Parser } from './common/parser/formats/rss20-parser';
+import { Atom10Parser } from './common/parser/formats/atom10-parser';
+import { FeedCrawler } from './feed-crawler';
 
 container.registerSingleton<DatabaseConnection>(
   DEPENDENCY_SYMBOLS.DatabaseConnection,
@@ -39,6 +43,29 @@ container.registerSingleton<ClaudeService>(
   ClaudeService,
 );
 
-container.registerSingleton<RssParser>(DEPENDENCY_SYMBOLS.RssParser, RssParser);
+container.registerSingleton<ParserUtil>(
+  DEPENDENCY_SYMBOLS.ParserUtil,
+  ParserUtil,
+);
+
+container.registerSingleton<Rss20Parser>(
+  DEPENDENCY_SYMBOLS.Rss20Parser,
+  Rss20Parser,
+);
+
+container.registerSingleton<Atom10Parser>(
+  DEPENDENCY_SYMBOLS.Atom10Parser,
+  Atom10Parser,
+);
+
+container.registerSingleton<FeedParserManager>(
+  DEPENDENCY_SYMBOLS.FeedParserManager,
+  FeedParserManager,
+);
+
+container.registerSingleton<FeedCrawler>(
+  DEPENDENCY_SYMBOLS.FeedCrawler,
+  FeedCrawler,
+);
 
 export { container };