diff --git a/Makefile b/Makefile index 2b8d4a04..c20030ad 100644 --- a/Makefile +++ b/Makefile @@ -54,3 +54,16 @@ clean-all: clean .PHONY: upgrade-deps upgrade-deps: npm run upgrade-deps + +# Manual test targets (requires CALS_GITHUB_TOKEN env var) +.PHONY: test-repos +test-repos: + CALS_GITHUB_TOKEN=$(CALS_GITHUB_TOKEN) node lib/cals-cli.mjs repos --org capralifecycle --compact + +.PHONY: test-clone +test-clone: + CALS_GITHUB_TOKEN=$(CALS_GITHUB_TOKEN) node lib/cals-cli.mjs clone --org capralifecycle --all | head -5 + +.PHONY: test-help +test-help: + node lib/cals-cli.mjs --help diff --git a/README.md b/README.md index b464127d..25a71e1a 100644 --- a/README.md +++ b/README.md @@ -13,52 +13,61 @@ It is recommended to use `npx` over global install to ensure you always run the latest version. If you install it globally remember to update it before running. -## Build +## Commands -Build and verify: +### Authentication -```sh -$ make # or "make build" +Set your GitHub token (will be stored in the OS keychain): + +```bash +cals auth ``` -## Contributing +### List repositories -This project uses [semantic release](https://semantic-release.gitbook.io/semantic-release/) -to automate releases and follows -[Git commit guidelines](https://github.com/angular/angular/blob/master/CONTRIBUTING.md#commit) -from the Angular project. +```bash +cals repos --org capralifecycle +cals repos --org capralifecycle --compact +cals repos --org capralifecycle --csv +``` + +### List repository groups + +```bash +cals groups --org capralifecycle +``` + +### Generate clone commands -Version numbers depend on the commit type and footers: https://github.com/semantic-release/commit-analyzer/blob/75c9c87c88772d7ded4ca9614852b42519e41931/lib/default-release-rules.js#L7-L12 +Generate clone commands (pipe to bash to execute): -## Goals of CLI +```bash +cals clone --org capralifecycle --all | bash +cals clone --org capralifecycle mygroup | bash +``` -- Provide an uniform way of consistently doing repeatable CALS tasks -- Provide simple guidelines to improve the experience for developers -- A tool that everybody uses and gets ownership of -- Automate repeatable CALS tasks as we go +### Sync repositories -## Ideas and future work +Pull latest changes for all repositories in a directory managed by a `.cals.yaml` manifest: -- Automate onboarding of people - - Granting access to various resources: AWS, GitHub, Confluence, JIRA, Slack, ... -- Automate offboarding of people -- Automate generation of new projects/resources - - Creating GitHub repos, giving permissions etc - - Slack channels - - AWS account and structure - - Checklist for manual processes -- AWS infrastructure management, e.g. scripts such as https://github.com/capralifecycle/rvr-aws-infrastructure/blob/master/rvr/create-stack.sh - - `cals aws ...` +```bash +cals sync +cals sync --clone # Prompt to clone missing repos +``` -### Snyk management +## Build -https://snyk.docs.apiary.io/reference/projects +Build and verify: -- [ ] Automatically set up project in Snyk -- [x] Report of which repos are in Snyk and which is not -- [ ] Detect active vs disabled projects in Snyk (no way through API now?) -- [x] Report issues in Snyk grouped by our projects +```sh +$ make # or "make build" +``` ## Contributing -This project doesn't currently accept contributions. For inquiries, please contact the maintainers at [Slack](https://liflig.slack.com/archives/C02T4KTPYS2). +This project uses [semantic release](https://semantic-release.gitbook.io/semantic-release/) +to automate releases and follows +[Git commit guidelines](https://github.com/angular/angular/blob/master/CONTRIBUTING.md#commit) +from the Angular project. + +For inquiries, please contact the maintainers at [Slack](https://liflig.slack.com/archives/C02T4KTPYS2). diff --git a/src/cli/commands/github/set-token.ts b/src/cli/commands/auth.ts similarity index 60% rename from src/cli/commands/github/set-token.ts rename to src/cli/commands/auth.ts index a9d35756..335ef412 100644 --- a/src/cli/commands/github/set-token.ts +++ b/src/cli/commands/auth.ts @@ -1,9 +1,9 @@ import type { CommandModule } from "yargs" -import { GitHubTokenCliProvider } from "../../../github/token" -import { type Reporter, readInput } from "../../reporter" -import { createReporter } from "../../util" +import { GitHubTokenCliProvider } from "../../github/token" +import { type Reporter, readInput } from "../reporter" +import { createReporter } from "../util" -async function setToken({ +async function authenticate({ reporter, token, tokenProvider, @@ -18,26 +18,26 @@ async function setToken({ "https://github.com/settings/tokens/new?scopes=repo:status,read:repo_hook", ) const inputToken = await readInput({ - prompt: "Enter new GitHub API token: ", + prompt: "Enter GitHub token: ", silent: true, }) token = inputToken } await tokenProvider.setToken(token) - reporter.info("Token saved") + reporter.info("Token saved to keychain") } const command: CommandModule = { - command: "set-token", - describe: "Set GitHub token for API calls", + command: "auth [token]", + describe: "Authenticate with GitHub", builder: (yargs) => yargs.positional("token", { - describe: - "Token. If not provided it will be requested as input. Can be generated at https://github.com/settings/tokens/new?scopes=repo:status,read:repo_hook", + describe: "GitHub token (prompted if not provided)", + type: "string", }), handler: async (argv) => { - await setToken({ + await authenticate({ reporter: createReporter(), token: argv.token as string | undefined, tokenProvider: new GitHubTokenCliProvider(), diff --git a/src/cli/commands/github/generate-clone-commands.ts b/src/cli/commands/clone.ts similarity index 61% rename from src/cli/commands/github/generate-clone-commands.ts rename to src/cli/commands/clone.ts index d2e9f895..5ef1e6ad 100644 --- a/src/cli/commands/github/generate-clone-commands.ts +++ b/src/cli/commands/clone.ts @@ -3,32 +3,28 @@ import path from "node:path" import process from "node:process" import yargs, { type CommandModule } from "yargs" import { hideBin } from "yargs/helpers" -import type { Config } from "../../../config" -import { createGitHubService, type GitHubService } from "../../../github" -import { getGroupedRepos, includesTopic } from "../../../github/util" -import type { Reporter } from "../../reporter" -import { createCacheProvider, createConfig, createReporter } from "../../util" +import type { Config } from "../../config" +import { createGitHubService, type GitHubService } from "../../github" +import { getGroupedRepos, includesTopic } from "../../github/util" +import { createCacheProvider, createConfig } from "../util" async function generateCloneCommands({ - reporter, config, github, org, ...opt }: { - reporter: Reporter config: Config github: GitHubService all: boolean - excludeExisting: boolean + skipCloned: boolean group: string | undefined includeArchived: boolean - listGroups: boolean name: string | undefined topic: string | undefined org: string }) { - if (!opt.listGroups && !opt.all && opt.group === undefined) { + if (!opt.all && opt.group === undefined) { yargs(hideBin(process.argv)).showHelp() return } @@ -36,16 +32,9 @@ async function generateCloneCommands({ const repos = await github.getOrgRepoList({ org }) const groups = getGroupedRepos(repos) - if (opt.listGroups) { - groups.forEach((it) => { - reporter.log(it.name) - }) - return - } - - groups.forEach((group) => { + for (const group of groups) { if (opt.group !== undefined && opt.group !== group.name) { - return + continue } group.items @@ -54,39 +43,35 @@ async function generateCloneCommands({ .filter((it) => opt.topic === undefined || includesTopic(it, opt.topic)) .filter( (it) => - !opt.excludeExisting || - !fs.existsSync(path.resolve(config.cwd, it.name)), + !opt.skipCloned || !fs.existsSync(path.resolve(config.cwd, it.name)), ) .forEach((repo) => { // The output of this is used to pipe into e.g. bash. - // We cannot use reporter.log as it adds additional characters. process.stdout.write( `[ ! -e "${repo.name}" ] && git clone ${repo.sshUrl}\n`, ) }) - }) + } } const command: CommandModule = { - command: "generate-clone-commands", - describe: "Generate shell commands to clone GitHub repos for an organization", + command: "clone [group]", + describe: "Generate git clone commands (pipe to bash to execute)", builder: (yargs) => yargs .positional("group", { - describe: "Group to generate commands for", + describe: "Clone only repos in this group", + type: "string", }) .options("org", { - demandOption: true, - describe: "Specify GitHub organization", + alias: "o", + default: "capralifecycle", + requiresArg: true, + describe: "GitHub organization", type: "string", }) .option("all", { - describe: "Use all groups", - type: "boolean", - }) - .option("list-groups", { - alias: "l", - describe: "List available groups", + describe: "Clone all repos", type: "boolean", }) .option("include-archived", { @@ -97,32 +82,32 @@ const command: CommandModule = { .option("name", { describe: "Filter to include the specified name", type: "string", + requiresArg: true, }) .option("topic", { alias: "t", describe: "Filter by specific topic", type: "string", + requiresArg: true, }) - .option("exclude-existing", { - alias: "x", - describe: "Exclude if existing in working directory", + .option("skip-cloned", { + alias: "s", + describe: "Skip repos already cloned in working directory", type: "boolean", }), handler: async (argv) => { const config = createConfig() return generateCloneCommands({ - reporter: createReporter(), config, github: await createGitHubService({ cache: createCacheProvider(config, argv), }), all: !!argv.all, - listGroups: !!argv["list-groups"], includeArchived: !!argv["include-archived"], name: argv.name as string | undefined, topic: argv.topic as string | undefined, - excludeExisting: !!argv["exclude-existing"], + skipCloned: !!argv["skip-cloned"], group: argv.group as string | undefined, org: argv.org as string, }) diff --git a/src/cli/commands/github.ts b/src/cli/commands/github.ts deleted file mode 100644 index 525196d9..00000000 --- a/src/cli/commands/github.ts +++ /dev/null @@ -1,40 +0,0 @@ -import process from "node:process" -import yargs, { type CommandModule } from "yargs" -import { hideBin } from "yargs/helpers" -import generateCloneCommands from "./github/generate-clone-commands" -import listRepos from "./github/list-repos" -import setToken from "./github/set-token" -import sync from "./github/sync" - -const command: CommandModule = { - command: "github", - describe: "Integration with GitHub", - builder: (yargs) => - yargs - .command(generateCloneCommands) - .command(listRepos) - .command(setToken) - .command(sync) - .demandCommand() - .usage(`cals github - -Notes: - Before doing anything against GitHub you need to configure a token - used for authentication. The following command will ask for a token - and provide a link to generate one: - $ cals github set-token - - Quick clone all repos: - $ cals github generate-clone-commands --org capralifecycle --all -x | bash - - And for a specific project: - $ cals github generate-clone-commands --org capralifecycle -x buildtools | bash - - Some responses are cached for some time. Use the --validate-cache - option to avoid stale cache.`), - handler: () => { - yargs(hideBin(process.argv)).showHelp() - }, -} - -export default command diff --git a/src/cli/commands/groups.ts b/src/cli/commands/groups.ts new file mode 100644 index 00000000..d047f0ed --- /dev/null +++ b/src/cli/commands/groups.ts @@ -0,0 +1,33 @@ +import type { CommandModule } from "yargs" +import { createGitHubService } from "../../github" +import { getGroupedRepos } from "../../github/util" +import { createCacheProvider, createConfig, createReporter } from "../util" + +const command: CommandModule = { + command: "groups", + describe: "List available repository groups in a GitHub organization", + builder: (yargs) => + yargs.options("org", { + alias: "o", + default: "capralifecycle", + requiresArg: true, + describe: "GitHub organization", + type: "string", + }), + handler: async (argv) => { + const config = createConfig() + const reporter = createReporter() + const github = await createGitHubService({ + cache: createCacheProvider(config, argv), + }) + + const repos = await github.getOrgRepoList({ org: argv.org as string }) + const groups = getGroupedRepos(repos) + + for (const group of groups) { + reporter.log(group.name) + } + }, +} + +export default command diff --git a/src/cli/commands/github/list-repos.ts b/src/cli/commands/repos.ts similarity index 90% rename from src/cli/commands/github/list-repos.ts rename to src/cli/commands/repos.ts index 11042d9e..7f37feb0 100644 --- a/src/cli/commands/github/list-repos.ts +++ b/src/cli/commands/repos.ts @@ -1,10 +1,10 @@ import process from "node:process" import type { CommandModule } from "yargs" -import { createGitHubService, type GitHubService } from "../../../github" -import type { Repo } from "../../../github/types" -import { getGroup, getGroupedRepos, includesTopic } from "../../../github/util" -import type { Reporter } from "../../reporter" -import { createCacheProvider, createConfig, createReporter } from "../../util" +import { createGitHubService, type GitHubService } from "../../github" +import type { Repo } from "../../github/types" +import { getGroup, getGroupedRepos, includesTopic } from "../../github/util" +import type { Reporter } from "../reporter" +import { createCacheProvider, createConfig, createReporter } from "../util" function getReposMissingGroup(repos: Repo[]) { return repos.filter((it) => getGroup(it) === null) @@ -131,13 +131,15 @@ async function listRepos({ } const command: CommandModule = { - command: "list-repos", - describe: "List Git repos for a GitHub organization", + command: "repos", + describe: "List repositories in a GitHub organization", builder: (yargs) => yargs .options("org", { - required: true, - describe: "Specify GitHub organization", + alias: "o", + default: "capralifecycle", + requiresArg: true, + describe: "GitHub organization", type: "string", }) .option("include-archived", { @@ -157,11 +159,13 @@ const command: CommandModule = { .option("name", { describe: "Filter to include the specified name", type: "string", + requiresArg: true, }) .option("topic", { alias: "t", describe: "Filter by specific topic", type: "string", + requiresArg: true, }), handler: async (argv) => { const config = createConfig() diff --git a/src/cli/commands/github/sync.ts b/src/cli/commands/sync.ts similarity index 94% rename from src/cli/commands/github/sync.ts rename to src/cli/commands/sync.ts index 83be7953..416cf56d 100644 --- a/src/cli/commands/github/sync.ts +++ b/src/cli/commands/sync.ts @@ -5,21 +5,18 @@ import { findUp } from "find-up" import yaml from "js-yaml" import pLimit from "p-limit" import type { CommandModule } from "yargs" -import type { Config } from "../../../config" -import { DefinitionFile, getRepos } from "../../../definition" +import type { Config } from "../../config" +import { DefinitionFile, getRepos } from "../../definition" import type { Definition, DefinitionRepo, GetReposResponse, -} from "../../../definition/types" -import { CloneType, GitRepo, type UpdateResult } from "../../../git/GitRepo" -import { getCompareLink } from "../../../git/util" -import { - createGitHubService, - type GitHubService, -} from "../../../github/service" -import { type Reporter, readInput } from "../../reporter" -import { createCacheProvider, createConfig, createReporter } from "../../util" +} from "../../definition/types" +import { CloneType, GitRepo, type UpdateResult } from "../../git/GitRepo" +import { getCompareLink } from "../../git/util" +import { createGitHubService, type GitHubService } from "../../github" +import { type Reporter, readInput } from "../reporter" +import { createCacheProvider, createConfig, createReporter } from "../util" const CALS_YAML = ".cals.yaml" const CALS_LOG = ".cals.log" @@ -374,15 +371,15 @@ async function sync({ github, cals, rootdir, - askClone, - askMove, + clone, + move, }: { reporter: Reporter github: GitHubService cals: CalsManifest rootdir: string - askClone: boolean - askMove: boolean + clone: boolean + move: boolean }) { const { expectedRepos, definitionRepo } = await getExpectedRepos( reporter, @@ -464,8 +461,8 @@ async function sync({ reporter.info(` ${it.actualRelpath} -> ${getRelpath(it)}`) } - if (!askMove) { - reporter.info("To move these repos on disk add --ask-move option") + if (!move) { + reporter.info("To move these repos on disk add --move option") } else { const shouldMove = await askMoveConfirm() if (shouldMove) { @@ -506,8 +503,8 @@ async function sync({ reporter.info(` ${it.id}`) } - if (!askClone) { - reporter.info("To clone these repos add --ask-clone option for dialog") + if (!clone) { + reporter.info("To clone these repos add --clone option") } else { reporter.info( "You must already have working credentials for GitHub set up for clone to work", @@ -572,16 +569,17 @@ const command: CommandModule = { describe: "Sync repositories for working directory", builder: (yargs) => yargs - .option("ask-clone", { + .option("clone", { alias: "c", - describe: "Ask to clone new missing repos", + describe: "Prompt to clone missing repos", type: "boolean", }) - .option("ask-move", { - describe: "Ask to actual move renamed repos", + .option("move", { + alias: "m", + describe: "Prompt to move renamed repos", type: "boolean", }) - .usage(`cals github sync + .usage(`cals sync Synchronize all checked out GitHub repositories within the working directory grouped by the project in the resource definition file. The command can also @@ -631,8 +629,8 @@ will be stored there.`), github, cals, rootdir: dir, - askClone: !!argv["ask-clone"], - askMove: !!argv["ask-move"], + clone: !!argv.clone, + move: !!argv.move, }) }, } diff --git a/src/cli/commands/topics.ts b/src/cli/commands/topics.ts new file mode 100644 index 00000000..dc7dfeb5 --- /dev/null +++ b/src/cli/commands/topics.ts @@ -0,0 +1,42 @@ +import type { CommandModule } from "yargs" +import { createGitHubService } from "../../github" +import { createCacheProvider, createConfig, createReporter } from "../util" + +const command: CommandModule = { + command: "topics", + describe: "List customer topics in a GitHub organization", + builder: (yargs) => + yargs.options("org", { + alias: "o", + default: "capralifecycle", + requiresArg: true, + describe: "GitHub organization", + type: "string", + }), + handler: async (argv) => { + const config = createConfig() + const reporter = createReporter() + const github = await createGitHubService({ + cache: createCacheProvider(config, argv), + }) + + const repos = await github.getOrgRepoList({ org: argv.org as string }) + + const topics = new Set() + for (const repo of repos) { + for (const edge of repo.repositoryTopics.edges) { + const name = edge.node.topic.name + if (name.startsWith("customer-")) { + topics.add(name) + } + } + } + + const sorted = [...topics].sort((a, b) => a.localeCompare(b)) + for (const topic of sorted) { + reporter.log(topic) + } + }, +} + +export default command diff --git a/src/cli/index.ts b/src/cli/index.ts index cb37ecb0..36b943e8 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -2,7 +2,12 @@ import process from "node:process" import yargs from "yargs" import { hideBin } from "yargs/helpers" import { engines, version } from "../../package.json" -import github from "./commands/github" +import auth from "./commands/auth" +import clone from "./commands/clone" +import groups from "./commands/groups" +import repos from "./commands/repos" +import sync from "./commands/sync" +import topics from "./commands/topics" declare const BUILD_TIMESTAMP: string @@ -32,16 +37,34 @@ export async function main(): Promise { } await yargs(hideBin(process.argv)) - .usage(`cals-cli v${version} (build: ${BUILD_TIMESTAMP})`) + .usage(`cals v${version} (build: ${BUILD_TIMESTAMP}) + +A CLI for managing GitHub repositories. + +Before using, authenticate with: cals auth`) .scriptName("cals") .locale("en") + .strict() + .strictCommands() + .strictOptions() .help("help") - .command(github) + .command(auth) + .command(clone) + .command(groups) + .command(repos) + .command(sync) + .command(topics) .version(version) .demandCommand() - .option("validate-cache", { - describe: "Only read from cache if validated against server", + .option("no-cache", { + describe: "Bypass cache and fetch fresh data", type: "boolean", }) + .example("cals auth", "Set GitHub token") + .example("cals repos", "List repositories") + .example("cals groups", "List repository groups") + .example("cals topics", "List customer topics") + .example("cals clone --all | bash", "Clone all repos") + .example("cals sync", "Pull latest changes") .parse() } diff --git a/src/cli/reporter.ts b/src/cli/reporter.ts index dfd5b11f..991ccfd8 100644 --- a/src/cli/reporter.ts +++ b/src/cli/reporter.ts @@ -14,16 +14,57 @@ export async function readInput(options: { silent?: boolean timeout?: number }): Promise { + process.stdout.write(options.prompt) + + // For silent mode, read character by character with raw mode to hide input + if (options.silent && process.stdin.isTTY) { + return new Promise((resolve, reject) => { + let input = "" + process.stdin.setRawMode(true) + process.stdin.resume() + process.stdin.setEncoding("utf8") + + const timer = options.timeout + ? setTimeout(() => { + cleanup() + reject(new Error("Input timed out")) + }, options.timeout) + : null + + const cleanup = () => { + process.stdin.setRawMode(false) + process.stdin.pause() + process.stdin.removeListener("data", onData) + if (timer) clearTimeout(timer) + } + + const onData = (char: string) => { + if (char === "\r" || char === "\n") { + cleanup() + process.stdout.write("\n") + resolve(input) + } else if (char === "\u0003") { + // Ctrl+C + cleanup() + process.exit(1) + } else if (char === "\u007F" || char === "\b") { + // Backspace + input = input.slice(0, -1) + } else { + input += char + } + } + + process.stdin.on("data", onData) + }) + } + + // Normal (non-silent) mode const rl = readline.createInterface({ input: process.stdin, output: process.stdout, }) - if (options.silent) { - // Mute output for password entry - ;(rl as any)._writeToOutput = () => {} - } - return new Promise((resolve, reject) => { const timer = options.timeout ? setTimeout(() => { @@ -32,12 +73,9 @@ export async function readInput(options: { }, options.timeout) : null - rl.question(options.prompt, (answer) => { + rl.question("", (answer) => { if (timer) clearTimeout(timer) rl.close() - if (options.silent) { - process.stdout.write("\n") - } resolve(answer) }) }) diff --git a/src/cli/util.ts b/src/cli/util.ts index 3f4a075e..ef09d26d 100644 --- a/src/cli/util.ts +++ b/src/cli/util.ts @@ -12,7 +12,7 @@ export function createCacheProvider( ): CacheProvider { const cache = new CacheProvider(config) - if (argv.validateCache === true) { + if (argv.noCache === true) { cache.mustValidate = true }