feat: Add new-post-checker lambda

Author: junah201

.github/workflows/main.yml

@@ -25,6 +25,7 @@ jobs:
             VH__add-bookmark-blog,
             VH__delete-bookmark-blog,
             VH__get-posts,
+            VH__new-post-checker,
           ]
         include:
           - lambda: VH__get-blog
@@ -51,6 +52,8 @@ jobs:
             path: ./backend/api/delete_bookmark_blog
           - lambda: VH__get-posts
             path: ./backend/api/get_posts
+          - lambda: VH__new-post-checker
+            path: ./backend/cron/new_post
 
     uses: junah201/velog-helper/.github/workflows/deploy.yml@main
     with:

backend/common/models.py

@@ -117,7 +117,7 @@ class Post(Base):
         nullable=False
     )
     short_description = Column(
-        String(200),
+        String(300),
         nullable=True,
         default=None
     )

backend/cron/edit_post/lambda_function.py

@@ -0,0 +1,11 @@
+import new_post
+import asyncio
+
+
+def lambda_handler(event, context):
+    asyncio.run(new_post.update_new_post())
+
+    return {
+        'statusCode': 200,
+        'body': "Success"
+    }

backend/cron/new_post/crawler.py

@@ -0,0 +1,43 @@
+import aiohttp
+from typing import List
+
+VELOG_API_URL = "https://v2.velog.io/graphql"
+
+get_new_posts_query = """
+query Posts($cursor: ID, $username: String, $temp_only: Boolean, $limit: Int) {
+    posts(cursor: $cursor, username: $username, temp_only: $temp_only, limit: $limit) {
+        id
+        title
+        short_description
+        user {
+            username
+            profile {
+                thumbnail
+            }
+        }
+        url_slug
+        released_at
+        updated_at
+    }
+}
+"""
+
+
+async def get_new_posts(username: str, limit: int = 10, return_type: str = "List") -> List[dict]:
+    async with aiohttp.ClientSession() as session:
+        async with session.post(VELOG_API_URL, json={"query": get_new_posts_query, "variables": {"username": username, "limit": limit}}) as resp:
+            assert resp.status == 200
+            data = await resp.json()
+
+            if return_type == "List":
+                return data["data"]["posts"]
+
+            if return_type == "Dict":
+                result = {}
+
+                for post in data["data"]["posts"]:
+                    result[post["id"]] = post
+
+                return result
+
+            return data["data"]["posts"]

backend/cron/new_post/discord_logging.py

@@ -0,0 +1,45 @@
+"""
+디스코드 채널에 웹훅을 이용해서 로깅
+"""
+
+from typing import List
+import aiohttp
+import datetime
+import os
+
+DISCORD_WEBHOOKS_NEW_POST_UPLOAD_LOG = os.environ.get(
+    "DISCORD_WEBHOOKS_NEW_POST_UPLOAD_LOG")
+
+
+color = {
+    "green": 0x2ECC71,
+}
+
+
+async def logging_new_post_upload(total_updated_blog_cnt: int, total_updated_posts: List[str]):
+    async with aiohttp.ClientSession() as session:
+        data = {
+            "content": "",
+            "embeds": [
+                {
+                    "title": "새 글 업데이트 로그",
+                    "description": f"블로그 : `{total_updated_blog_cnt}`개\n포스트 : `{len(total_updated_posts)}`개",
+                    "color": color["green"],
+                    "footer": {
+                        "text": f"{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+                    }
+                }
+            ]
+        }
+
+        if total_updated_posts:
+            tmp = ""
+            print(total_updated_posts)
+            for post in total_updated_posts:
+                tmp += post
+            data["embeds"][0]["description"] += f"\n\n```{tmp}```"
+
+        await session.post(
+            url=DISCORD_WEBHOOKS_NEW_POST_UPLOAD_LOG,
+            json=data
+        )

backend/cron/new_post/lambda_function.py

@@ -0,0 +1,11 @@
+import new_post
+import asyncio
+
+
+def lambda_handler(event, context):
+    asyncio.run(new_post.update_new_post())
+
+    return {
+        'statusCode': 200,
+        'body': "Success"
+    }

backend/cron/new_post/mail.py

@@ -0,0 +1,43 @@
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+import os
+from models import Post
+
+BACKEND_SERVER_URL = os.environ.get("BACKEND_SERVER_URL", None)
+MAIL_SENDER = os.environ.get("MAIL_SENDER", None)
+MAIL_PASSWARD = os.environ.get("MAIL_PASSWARD", None)
+
+env = Environment(
+    loader=FileSystemLoader('./'),
+    autoescape=select_autoescape(['html']),
+)
+
+
+def send_new_post_notice_email(receiver_address: str, post: Post, user_id: str) -> None:
+    new_post_notice_template = env.get_template("new_post_notice.html")
+
+    message = MIMEMultipart()
+    message['From'] = MAIL_SENDER
+    message['To'] = receiver_address
+    message['Subject'] = f"{post.title} | 새 글 알림"
+    message.attach(MIMEText(new_post_notice_template.render(
+        user=post.blog_id,
+        title=post.title,
+        link=post.link,
+        BACKEND_SERVER_URL=BACKEND_SERVER_URL,
+        user_id=user_id,
+        short_description=post.short_description or "이 글의 요약을 가져오지 못했습니다. (2022.11.21 이전 글 일 가능성이 있습니다.)",
+        user_img=post.blog_img,
+        released_year=post.created_at.year,
+        released_month=post.created_at.month,
+        released_day=post.created_at.day
+    ), 'html'))
+    session = smtplib.SMTP('smtp.gmail.com', 587)
+    session.starttls()
+    session.login(MAIL_SENDER, MAIL_PASSWARD)
+    text = message.as_string()
+    session.sendmail(MAIL_SENDER, receiver_address, text)
+    session.quit()
+    print(f'Mail Sent (new_post) "{receiver_address}" "{post.title}"')

backend/cron/new_post/new_post.py

@@ -0,0 +1,98 @@
+from sqlalchemy.orm import Session
+import models
+import database
+from crawler import get_new_posts
+from typing import List, Tuple
+import discord_logging
+import datetime
+import mail
+
+VELOG_DEFAULT_PROFILE_IMG = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAYAAADDPmHLAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAASbSURBVHgB7Z0tTytBFIYP914BDiQ4cIADB0EhwYFE8ifq7g/hJ2CRSCQ4kOCobF3ruHk3maS5aSnbdnfPOe/7JE0oCTvTnmc+dvbMsNbr9b5M0PLLBDUSgBwJQI4EIEcCkCMByJEA5EgAciQAORKAHAlAjgQgRwKQIwHIkQDkSAByJAA5EoAcCUCOBCBHApAjAciRAORIAHIkADkSgBwJQI4EIEcCkCMByJEA5EgAciQAOX+MhPX1dTs+Prbt7W3b3d21jY2N6ndgPB7bYDCw4XBor6+v9vHxUb1nIL0Ae3t7dn5+XgV9FhABYuC1v79f/Q4SPD8/28vLi2UmrQA/Cfx34O/wwjXu7u7S9gi/z87O/loyELTr62vb2tqyZcFQcXp6Wv2MXiEb6SaBCDwEWDVFqmykEgABOjo6sqbAtbNJkEaAi4uLRoNfQBmXl5eWhRQCIChlnG6Dk5OTVstrkvACYKLXxJg/D5RZ1hEiE14ABGIVs/26IPgZeoHQAiDwbYz7s4AA0XuB0AIsusizKsrycmRCC+Dhyz84OLDIhBUAra/rHgCgDpGHgbAC7OzsmBc81aUuYQXY3Nw0L3iqS13CCtDFrd8sPNWlLsoIIkcCkBNWAE8JGpGTRcIKgPw9L3iqS13CCvD5+Wle8FSXuoQVAJm8HlK0UAfUJSqhJ4Fvb2/WNcgcjkxoAfDld936oieKhhYAwX96erKuwJ6B6Oni4dcBIEAXvQAC//j4aNEJLwCC30UgUGaGzSIpVgLRC7Q5FKCsLFvG0iwFPzw8tBIUlIGyspDqWcD9/X2jEuDaKCMT6R4GIUBNzAlwzWzBByl3ByNYaK23t7dLP6vHfT6u9/7+bhlZ6/V6X5YYpI0jebRu/mD2wBfSHxCBngAv9ASQ4PDwsErhwvvJE0JGo1EV9H6/72KFsS1SCDAZyFngnh2vVUwSUV4WQUILULZnlR06aMGYqDW1QDN56khZho6+Ghh2DoBgXF1dTZ3koZWvcqWubECdtg0NZUQ+QiakAGjxOA9gHhABj4wXeWyMHgX5/j85Zwi9AXoeD4+n6xJOAASk7nbwkjyCGT0meXg/mcWDYOMsIJwShtaO3mWRHT/odaINCaHmAIsEHyCQOP6tHAHXFKVukSQIsxK4aPDbBnWMdG5ACAHwhUYIfgHzEwwjEXAvQFdHwCzLzc1NiC1jrgXA2I31/Ijbr1HnCEfKuRagq/N/VgXuJLzPB9wKgMBnOITJu8RuBUDXnwHvQ4FLAbDkGrnr/x8MBV7vClwKEHHWPw+vn8mdANlaf8FrL+BOgIytv+Dxs7kSAC0kY+sveOwFXAnQ5bGvbdH0A6m6uBLAw8GPTePtaFk3AmTv/gtYF/A0DLgRgKH1Fzx9VjcCIBuHBU89nRsBkKrFgqfNJm5SwpBGVc7fz/CvWKZRUsk9bS1PvzVMfI+OiiVHApAjAciRAORIAHIkADkSgBwJQI4EIEcCkCMByJEA5EgAciQAORKAHAlAjgQgRwKQIwHIkQDkSAByJAA5EoAcCUCOBCBHApAjAciRAORIAHIkADkSgBwJQI4EIOcfGjV2tEfztqEAAAAASUVORK5CYII="
+TZ_KST = datetime.timezone(datetime.timedelta(hours=9))
+
+
+def UTC_to_KST(utc_time: datetime.datetime) -> datetime.datetime:
+    kst_time = utc_time + datetime.timedelta(hours=9)
+    # kst_time = kst_time.replace(tzinfo=TZ_KST)
+    return kst_time
+
+
+async def update_new_post_by_blog(db: Session, blog: models.Blog, limit: int = 10, is_init: bool = False) -> List[str]:
+    updated_posts: List[str] = list()
+
+    posts = await get_new_posts(username=blog.id, limit=limit, return_type="List")
+
+    if is_init:
+        blog.last_uploaded_at = datetime.datetime(2005, 2, 1)
+
+    for post in reversed(posts):
+        post_uploaded_at = UTC_to_KST(datetime.datetime.strptime(
+            post["released_at"][:19], "%Y-%m-%dT%H:%M:%S"))
+
+        print(post_uploaded_at, blog.last_uploaded_at)
+        if post_uploaded_at <= blog.last_uploaded_at:
+            continue
+
+        if db.query(models.Post).filter(models.Post.id == post["id"]).first():
+            continue
+
+        updated_posts.append(f"{blog.id} - {post['title']}")
+
+        # DB에 추가
+        db_post = models.Post(
+            id=post["id"],
+            title=post["title"],
+            blog_id=post["user"]["username"],
+            blog_img=post["user"]["profile"]["thumbnail"] if post["user"]["profile"]["thumbnail"] else VELOG_DEFAULT_PROFILE_IMG,
+            link=post["url_slug"],
+            short_description=post["short_description"],
+            created_at=post_uploaded_at,
+            updated_at=UTC_to_KST(datetime.datetime.strptime(
+                post["updated_at"][:19], "%Y-%m-%dT%H:%M:%S")),
+        )
+
+        db.add(db_post)
+        db.commit()
+        db.refresh(db_post)
+
+        db_users = db.query(models.Bookmark).filter(
+            models.Bookmark.blog_id == db_post.blog_id).all()
+
+        for bookmarked_user in db_users:
+            db_user = db.query(models.User).filter(
+                models.User.id == bookmarked_user.user_id).first()
+            if not db_user.email:
+                continue
+
+            if db_user.is_subscribed and not is_init:
+                mail.send_new_post_notice_email(
+                    receiver_address=db_user.email, post=db_post, user_id=db_user.id)
+
+    last_uploaded_at = datetime.datetime.now()
+    if posts:
+        last_uploaded_at = UTC_to_KST(datetime.datetime.strptime(
+            posts[0]["released_at"][:19], "%Y-%m-%dT%H:%M:%S"))
+
+    db.query(models.Blog).filter(
+        models.Blog.id == blog.id).update(
+            {"last_uploaded_at": last_uploaded_at,
+             "updated_at": str(datetime.datetime.now())})
+    db.commit()
+
+    return updated_posts
+
+
+async def update_new_post() -> None:
+    db = next(database.get_db())
+
+    total_updated_blog_cnt: int = 0
+    total_updated_posts: List[str] = list()
+
+    db_blogs = db.query(models.Blog).all()
+    for blog in db_blogs:
+        updated_posts = await update_new_post_by_blog(db=db, blog=blog, limit=10)
+        if updated_posts:
+            total_updated_blog_cnt += 1
+            total_updated_posts.extend(updated_posts)
+
+    await discord_logging.logging_new_post_upload(total_updated_blog_cnt, total_updated_posts)