feat: fetch grammar from upstream

uses head.json to keep track of what commit hash from upstream is published

Author: devanshj

.gitignore

@@ -1,8 +1,4 @@
 node_modules
 dist
-src/parser/JavaParser.ts
-src/parser/JavaParserListener.ts
-src/parser/JavaParserVisitor.ts
-src/parser/JavaLexer.ts
-src/parser/JavaContexts.ts
-src/parser/*.interp
+src/parser/*
+!src/parser/.gitkeep

build.ts

@@ -0,0 +1,156 @@
+import { promises as fs } from 'fs';
+import rimraf from 'rimraf';
+import fetch from 'node-fetch';
+import path from 'path';
+import { exec } from 'child_process';
+import { once } from 'events';
+import { EOL } from 'os';
+import { promisify } from 'util';
+
+const files = [
+  'JavaParser.g4',
+  'JavaLexer.g4'
+];
+
+const main = () =>
+  withLog(
+    'Checking if head is stale... ',
+    getIsStale(),
+    isStale => isStale ? 'Stale' : 'Up-to date'
+  )
+  .then(isStale => isStale || process.argv.includes('--force'))
+  .then(shouldBuild =>
+    !shouldBuild
+      ? (console.log('Exiting, use --force to build anyway'), Promise.reject(terminationSignal))
+      : Promise.resolve()
+  )
+  .then(() => withLog('Fetching files from upstream... ', getFiles()))
+  .then(files => withLog('Writing files... ', writeFiles(files)))
+  .then(() => withLog('Updating head.json... ', updateHead()))
+  .then(() => withLog('Generating parser...\n', writeParser()))
+  .then(() => withLog('Generating contexts... ', writeParserContexts()))
+  .then(() => withLog('Compiling typescript files... ', writeJavascript()))
+  .then(() => console.log('Build successful!'))
+  .catch(payload =>
+    payload === terminationSignal
+      ? Promise.resolve()
+      : Promise.reject(payload)
+  )
+
+const getIsStale = () => 
+  Promise.all([getHead(), getUpstreamHead()])
+  .then(([head, upstreamHead]) =>
+    files.some(file => head[file] !== upstreamHead[file])
+  )
+
+const getHead = () =>
+  fs.readFile(path.join(__dirname, 'src/head.json'), 'utf-8')
+  .then(JSON.parse) as Promise<{ [file: string]: string }>
+
+let upstreamHeadCache: { [file: string]: string } | undefined;
+const getUpstreamHead = () =>
+  upstreamHeadCache ? Promise.resolve(upstreamHeadCache) :
+  Promise.all(
+    files.map(file =>
+      fetch(`https://api.github.com/repos/antlr/grammars-v4/commits?path=java/java/${file}`)
+      .then(res => res.json())
+      .then(commits => ({ [file]: commits[0].sha as string }))
+    )
+  )
+  .then(mergeAll)
+  .then(upstreamHead => {
+    upstreamHeadCache = upstreamHead;
+    return Promise.resolve(upstreamHead);
+  });
+  
+const getFiles = () =>
+  Promise.all(
+    files.map(
+      file => 
+      fetch(`https://raw.githubusercontent.com/antlr/grammars-v4/master/java/java/${file}`)
+      .then(res => res.text())
+      .then(data => ({ [file]: data }))
+    )
+  )
+  .then(mergeAll)
+
+const writeFiles = (files: { [file: string]: string }) =>
+  Promise.all(
+    Object.entries(files)
+    .map(([file, data]) =>
+      fs.writeFile(path.join(__dirname, 'src/parser/', file), data)
+    )
+  )
+
+const updateHead = () =>
+  getUpstreamHead()
+  .then(head =>
+    fs.writeFile(
+      path.join(__dirname, 'src/head.json'),
+      JSON.stringify(head, null, '  ')
+    )
+  )
+
+const writeParser = () =>
+  execCommand(`${prependBinDir('antlr4ts')} -visitor -o src/parser -Xexact-output-dir src/parser/JavaLexer.g4 src/parser/JavaParser.g4`)
+
+const writeParserContexts = () =>
+  fs.readFile(path.join(__dirname, '/src/parser/JavaParserListener.ts'), 'utf-8')
+  .then(listenerSource =>
+    listenerSource
+    .split(EOL)
+    .map((l) => {
+      let matches = l.match(/import\s*\{\s*(.*Context)\s*\}.*/);
+      if (matches === null) return null;
+      return matches[1];
+    })
+    .filter((c) => c !== null)
+  )
+  .then(contexts => contexts.reduce((list, context) => list + `  ${context},${EOL}`, ''))
+  .then(exportList => `export {${EOL}${exportList}} from './JavaParser';`)
+  .then(contextsSource => fs.writeFile(path.join(__dirname, '/src/parser/JavaContexts.ts'), contextsSource));
+
+const writeJavascript = () => 
+  promisify(rimraf)(path.join(__dirname, "/dist"))
+  .then(() => execCommand(prependBinDir('tsc')));
+
+const withLog = <T>(
+  label: string,
+  promise: Promise<T>,
+  fulfilMessage: ((value: T) => string) = () => 'Done'
+) => {
+  process.stdout.write(label);
+  return promise
+  .then(value => {
+    process.stdout.write(fulfilMessage(value) + '\n')
+    return Promise.resolve(value);
+  })
+  .catch(error => {
+    process.stdout.write('Something went wrong\n');
+    return Promise.reject(error);
+  })
+}
+
+const terminationSignal = Symbol('terminationSignal');
+
+const execCommand = (command: string) => {
+  let childProcess = exec(command, { cwd: __dirname })
+  childProcess.stdout.pipe(process.stdout);
+  childProcess.stderr.pipe(process.stderr);
+
+  return (
+    once(childProcess, 'exit')
+    .then(([code]: [number]) => code === 0 ? Promise.resolve() : Promise.reject())
+  )
+}
+
+const prependBinDir = (p: string) =>
+  path.join(__dirname, "/node_modules/.bin/", p);
+
+type MergeAll = <T extends object[]>(xs: T) => UnionToIntersection<T[number]>;
+const mergeAll: MergeAll = xs => xs.reduce((m, x) => ({ ...m, ...x }), {}) as any;
+  
+type UnionToIntersection<U> = 
+  (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I : never
+
+main();

generate-contexts.js

@@ -21,26 +21,28 @@
     "dist"
   ],
   "scripts": {
-    "build": "rimraf dist && npm run generate:parser && npm run generate:contexts && tsc",
-    "format": "prettier --write src/**.ts **/*.json",
+    "build": "ts-node build.ts",
+    "format": "prettier --write build.ts src/**.ts **/*.json",
     "prepublish": "yarn build",
-    "generate:parser": "antlr4ts -visitor -o src/parser -Xexact-output-dir src/parser/JavaLexer.g4 src/parser/JavaParser.g4",
-    "generate:contexts": "node generate-contexts.js",
     "precommit": "lint-staged",
     "postcommit": "git update-index --again",
     "test": "jest"
   },
   "devDependencies": {
     "@types/jest": "^26.0.4",
     "@types/node": "^14.0.22",
+    "@types/node-fetch": "^2.5.7",
+    "@types/rimraf": "^3.0.0",
     "antlr4ts-cli": "^0.5.0-alpha.3",
-    "husky": "^0.14.3",
+    "husky": "^4.2.5",
     "jest": "^26.1.0",
-    "lint-staged": "^7.2.0",
-    "prettier": "^1.13.7",
-    "rimraf": "^2.6.2",
+    "lint-staged": "^10.2.11",
+    "node-fetch": "^2.6.0",
+    "prettier": "^2.0.5",
+    "rimraf": "^3.0.2",
     "ts-jest": "^26.1.1",
-    "tslint": "^5.10.0",
+    "ts-node": "^8.10.2",
+    "tslint": "^6.1.2",
     "typescript": "^3.9.6"
   },
   "dependencies": {

package.json

@@ -0,0 +1,4 @@
+{
+  "JavaParser.g4": null,
+  "JavaLexer.g4": null
+}