Add Delay & Fix Get Posts Error

Author: cjaewon

app.js

@@ -6,14 +6,19 @@ const program = new Command();
 
 program.version('1.0.0');
 program.option('-u, --username <username>', 'velog 유저이름');
+program.option('-d, --delay <ms>', '요청 딜레이 시간')
 
 program.parse(process.argv);
 
 !fs.existsSync('./backup') && fs.mkdirSync('./backup');
 !fs.existsSync('./backup/content') && fs.mkdirSync('./backup/content');
 !fs.existsSync('./backup/images') && fs.mkdirSync('./backup/images');
 
-const crawler = new Crawler(program.username);
+
+
+const crawler = new Crawler(program.username, { 
+  delay: program.delay || 0,
+ });
 
 console.log('📙 백업을 시작합니다 / velog-backup')
 crawler.parse()

crawler/index.js

@@ -5,16 +5,21 @@ const { join } = require('path');
 const { PostsQuery, PostQuery } = require('./query');
 
 class Crawler {
-  constructor(username) {
-    this.username = username;
+  constructor(username, { delay }) {
+    this.username = username; 
+
+    // options
+    this.delay = delay;
 
     this.__grahpqlURL = 'https://v2.velog.io/graphql';
   }
 
   async parse() {
     const posts = await this.getPosts();
 
-    posts.map(async(postInfo, i) => {
+    posts.map(async(postInfo, i) => { 
+      if (this.delay > 0) await new Promise(r => setTimeout(r, this.delay * i));
+
       let post = await this.getPost(postInfo.url_slug);
       post.body = await this.getImage(post.body);
 
@@ -26,6 +31,7 @@ class Crawler {
   async getPosts() {
     const url = `https://velog.io/@${this.username}`;
     let response;
+    let posts = [];
 
     try {
       await axios.get(url);
@@ -38,14 +44,21 @@ class Crawler {
       console.error(e);
     }
 
-    try {
-      response = await axios.post(this.__grahpqlURL, PostsQuery(this.username));
-    } catch(e) {
-      console.error(`⚠️  벨로그에서 글 목록을 가져오는데 실패했습니다. \n error = ${e}`);
-      process.exit(1);
+    while (true) {
+      try {
+        if (response && response.data.data.posts.length >= 20) {
+          response = await axios.post(this.__grahpqlURL, PostsQuery(this.username, posts[posts.length - 1].id));
+        } else {
+          response = await axios.post(this.__grahpqlURL, PostsQuery(this.username));
+        }
+      } catch(e) {
+        console.error(`⚠️  벨로그에서 글 목록을 가져오는데 실패했습니다. \n error = ${e}`);
+        process.exit(1);
+      }
+      
+      posts = [...posts, ...response.data.data.posts];
+      if (response.data.data.posts.length < 20) break;
     }
-  
-    const posts = response.data.data.posts;
 
     console.log(`✅ ${this.username}님의 모든 글(${posts.length} 개) 을 가져옴`);
 
@@ -89,7 +102,9 @@ class Crawler {
         method: 'get',
         url,
         responseType: 'stream'
-      }).then(resp => resp.data.pipe(fs.createWriteStream(path)));
+      })
+      .then(resp => resp.data.pipe(fs.createWriteStream(path)))
+      .catch(e => console.error(`⚠️ 이미지를 다운 받는데 오류가 발생했습니다 / url = ${url} , e = ${e}`));
 
       return `![](/images/${filename})`;
     });

crawler/query.js

@@ -1,8 +1,9 @@
-const PostsQuery = (username) => ({
+const PostsQuery = (username, cursor = null) => ({
   operationName:'Posts',
   variables: {
     username,
     tag: null,
+    cursor,
   },
 
   query: `query Posts($cursor: ID, $username: String, $temp_only: Boolean, $tag: String, $limit: Int) {