Simplify to only storing seed file seed URLs and add tests

The only information in seed files is the URL, so no need to
complicate things by storing more than that.

Author: tw4l

src/util/parseseeds.ts

@@ -30,9 +30,8 @@ export async function parseSeeds(
     seedFileDone &&
     crawlState
   ) {
-    for (const seed of params.state.seedFileSeeds) {
-      const scopedSeed: ScopedSeed = JSON.parse(seed);
-      await crawlState.addSeedFileSeed(scopedSeed);
+    for (const seedUrl of params.state.seedFileSeeds) {
+      await crawlState.addSeedFileSeed(seedUrl);
     }
   }
 
@@ -76,7 +75,7 @@ export async function parseSeeds(
       const scopedSeed = new ScopedSeed({ ...scopeOpts, ...newSeed });
       scopedSeeds.push(scopedSeed);
       if (params.seedFile && !seedFileDone && crawlState) {
-        await crawlState.addSeedFileSeed(scopedSeed);
+        await crawlState.addSeedFileSeed(scopedSeed.url);
         logger.debug(
           "Pushed seed file seed to Redis",
           { url: scopedSeed.url },
@@ -103,21 +102,21 @@ export async function parseSeeds(
 
   // If seed file was already successfully parsed, re-add seeds from Redis
   if (params.seedFile && seedFileDone && crawlState) {
-    const seedFileScopedSeeds = await crawlState.getSeedFileSeeds();
-    for (const seed of seedFileScopedSeeds) {
+    const seedFileSeedUrls = await crawlState.getSeedFileSeeds();
+    for (const seedUrl of seedFileSeedUrls) {
       logger.debug(
         "Pulled seed file seed from Redis",
-        { url: seed.url },
+        { url: seedUrl },
         "seedFile",
       );
       try {
-        const scopedSeed = new ScopedSeed({ ...scopeOpts, url: seed.url });
+        const scopedSeed = new ScopedSeed({ ...scopeOpts, url: seedUrl });
         scopedSeeds.push(scopedSeed);
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
       } catch (e: any) {
         logger.error("Failed to create seed from Redis", {
           error: e.toString(),
-          ...seed,
+          url: seedUrl,
         });
       }
     }

src/util/state.ts

@@ -743,7 +743,7 @@ return inx;
     const pending = await this.getPendingList();
     const failed = await this._iterListKeys(this.fkey, seen);
     const errors = await this.getErrorList();
-    const seedFileSeeds = await this._iterListKeys(this.seedFileSeedsKey, seen);
+    const seedFileSeeds = await this.getSeedFileSeeds();
     const extraSeeds = await this._iterListKeys(this.esKey, seen);
     const sitemapDone = await this.isSitemapDone();
     const seedFileDone = await this.isSeedFileDone();
@@ -1050,11 +1050,11 @@ return inx;
     return await this.redis.lpush(this.pageskey, JSON.stringify(data));
   }
 
-  async addSeedFileSeed(seed: ScopedSeed) {
-    const ret = await this.redis.sadd(this.seedFileSeedsMap, seed.url);
+  async addSeedFileSeed(url: string) {
+    const ret = await this.redis.sadd(this.seedFileSeedsMap, url);
     if (ret > 0) {
       // Push to end of list to keep seeds in order for ids
-      await this.redis.rpush(this.seedFileSeedsKey, JSON.stringify(seed));
+      await this.redis.rpush(this.seedFileSeedsKey, url);
     }
   }
 
@@ -1112,13 +1112,7 @@ return inx;
   }
 
   async getSeedFileSeeds() {
-    const seeds: ScopedSeed[] = [];
-
-    const res = await this.redis.lrange(this.seedFileSeedsKey, 0, -1);
-    for (const key of res) {
-      seeds.push(JSON.parse(key));
-    }
-    return seeds;
+    return await this.redis.lrange(this.seedFileSeedsKey, 0, -1);
   }
 
   async getExtraSeeds() {

tests/url_file_list.test.js

@@ -1,10 +1,23 @@
 import util from "util";
-import { spawn, exec as execCallback } from "child_process";
+import { spawn, execSync, exec as execCallback } from "child_process";
 import fs from "fs";
+import path from "path";
+import yaml from "js-yaml";
+import Redis from "ioredis";
 
 const exec = util.promisify(execCallback);
 
+const pagesFile = "test-crawls/collections/seed-file-restart-test/pages/pages.jsonl";
+const extraPagesFile = "test-crawls/collections/seed-file-restart-test/pages/extraPages.jsonl";
+
+const expectedSeedFileSeeds = [
+  "https://old.webrecorder.net/about/",
+  "https://specs.webrecorder.net/wacz/1.1.1/",
+  "https://old.webrecorder.net/faq"
+];
+
 let proc = null;
+let redisId = null;
 
 const DOCKER_HOST_NAME = process.env.DOCKER_HOST_NAME || "host.docker.internal";
 const TEST_HOST = `http://${DOCKER_HOST_NAME}:31502`;
@@ -20,6 +33,38 @@ afterAll(() => {
 });
 
 
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitContainerDone(containerId) {
+  // containerId is initially the full id, but docker ps
+  // only prints the short id (first 12 characters)
+  containerId = containerId.slice(0, 12);
+
+  while (true) {
+    try {
+      const res = execSync("docker ps -q", { encoding: "utf-8" });
+      if (res.indexOf(containerId) < 0) {
+        return;
+      }
+    } catch (e) {
+      console.error(e);
+    }
+    await sleep(500);
+  }
+}
+
+async function killContainer(containerId) {
+  try {
+    execSync(`docker kill -s SIGINT ${containerId}`);
+  } catch (e) {
+    return;
+  }
+
+  await waitContainerDone(containerId);
+}
+
 
 test("check that URLs in seed-list are crawled", async () => {
   try {
@@ -91,3 +136,173 @@ test("check that URLs in seed-list hosted at URL are crawled", async () => {
   }
   expect(foundSeedUrl).toBe(true);
 });
+
+
+let savedStateFile;
+let finished;
+
+test("start crawl from seed list and then interrupt and save state when seeds have been crawled", async () => {
+  let containerId = null;
+
+  try {
+    containerId = execSync(
+      `docker run -d -e CRAWL_ID=seedfiletest -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection seed-file-restart-test --seedFile "${TEST_HOST}/urlSeedFile.txt" --limit 10 --behaviors "" --exclude community --scopeType page --extraHops 1 --logging stats,debug`,
+      { encoding: "utf-8" },
+    );
+  } catch (error) {
+    console.log(error);
+  }
+
+  // remove existing pagesFile to support reentrancy
+  try {
+    fs.unlinkSync(pagesFile);
+  } catch (e) {
+    // ignore
+  }
+
+  while (true) {
+    try {
+      const pages = fs
+        .readFileSync(pagesFile, { encoding: "utf-8" })
+        .trim()
+        .split("\n");
+
+      if (pages.length >= 4) {
+        break;
+      }
+    } catch (e) {
+      // ignore
+    }
+
+    await sleep(500);
+  }
+
+  await killContainer(containerId);
+
+  const savedStates = fs.readdirSync(
+    "test-crawls/collections/seed-file-restart-test/crawls",
+  );
+  expect(savedStates.length > 0).toEqual(true);
+
+  savedStateFile = savedStates[savedStates.length - 1];
+});
+
+
+test("check saved state for seed file seeds", () => {
+  expect(savedStateFile).toBeTruthy();
+
+  const savedState = fs.readFileSync(
+    path.join("test-crawls/collections/seed-file-restart-test/crawls", savedStateFile),
+    "utf-8",
+  );
+
+  const saved = yaml.load(savedState);
+
+  const state = saved.state;
+  finished = state.finished;
+
+  const numDone = finished.length;
+  const numQueued = state.queued.length;
+
+  expect(!!state).toBe(true);
+  expect(numDone > 0).toEqual(true);
+  expect(numQueued > 0).toEqual(true);
+
+  const seedFileDone = state.seedFileDone;
+  expect(seedFileDone).toEqual(true);
+
+  const seedFileSeeds = state.seedFileSeeds;
+  expect(seedFileSeeds.length).toEqual(3);
+  for (const [index, seed] of seedFileSeeds.entries()) {
+    expect(seed).toEqual(expectedSeedFileSeeds[index]);
+  }
+});
+
+
+test("check seed file seed crawl finishes successfully after resuming from saved state", async () => {
+  let containerId = null;
+
+  const port = 36383;
+
+  try {
+    containerId = execSync(
+      `docker run -d -p ${port}:6379 -e CRAWL_ID=seedfiletest -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection seed-file-restart-test --debugAccessRedis --config /crawls/collections/seed-file-restart-test/crawls/${savedStateFile} --seedFile "${TEST_HOST}/urlSeedFile.txt" --limit 10 --behaviors "" --exclude community --scopeType page --extraHops 1 --logging stats,debug`,
+      { encoding: "utf-8" },
+    );
+  } catch (error) {
+    console.log(error);
+  }
+
+  await sleep(2000);
+
+  const redis = new Redis(`redis://127.0.0.1:${port}/0`, { lazyConnect: true, retryStrategy: () => null });
+
+  try {
+    await redis.connect({
+      maxRetriesPerRequest: 100,
+    });
+
+    await sleep(2000);
+
+    for (const url of finished) {
+      const res = await redis.sismember("seedfiletest:s", url);
+      expect(res).toBe(1);
+    }
+  } catch (e) {
+    console.log(e);
+  } finally {
+    await waitContainerDone(containerId);
+  }
+});
+
+test("ensure all pages were crawled", async () => {
+  const pages = fs
+    .readFileSync(pagesFile, { encoding: "utf-8" })
+    .trim()
+    .split("\n");
+
+  // first line is the header
+  expect(pages.length).toBe(4);
+
+  const extraPages = fs
+    .readFileSync(extraPagesFile, { encoding: "utf-8" })
+    .trim()
+    .split("\n");
+
+  // first line is the header
+  expect(extraPages.length).toBe(8);
+})
+
+
+test("ensure that seed file seeds were pulled from Redis on restart", async () => {
+  const logDir = "test-crawls/collections/seed-file-restart-test/logs/";
+  const logFiles = [];
+  fs.readdirSync(logDir).forEach((file) => {
+    if (file.endsWith(".log")) {
+      logFiles.push(path.join(logDir, file));
+    }
+  });
+
+  expect(logFiles.length).toBeGreaterThan(0);
+
+  const logFile = logFiles[logFiles.length - 1];
+  const log = fs.readFileSync(logFile, { encoding: "utf-8" }).trim();
+
+  expect(
+    log.indexOf(
+      '"logLevel":"debug","context":"seedFile","message":"Pulled seed file seed from Redis","details":{"url":"https://old.webrecorder.net/about/"}',
+    ) > 0,
+  ).toBe(true);
+
+  expect(
+    log.indexOf(
+      '"logLevel":"debug","context":"seedFile","message":"Pulled seed file seed from Redis","details":{"url":"https://specs.webrecorder.net/wacz/1.1.1/"}',
+    ) > 0,
+  ).toBe(true);
+
+  expect(
+    log.indexOf(
+      '"logLevel":"debug","context":"seedFile","message":"Pulled seed file seed from Redis","details":{"url":"https://old.webrecorder.net/faq"}',
+    ) > 0,
+  ).toBe(true);
+});