From 0568234e96f13bcc0fb6b28fd0935b5612fed9f7 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 19 Mar 2026 13:00:15 +0000 Subject: [PATCH 01/17] Add Azure DevOps to GitOps pipelines fixes #5491 --- .../20260318-01-EE-extend-gittoken.js | 16 + forge/ee/db/models/GitToken.js | 5 + forge/ee/db/models/PipelineStageGitRepo.js | 2 + forge/ee/db/views/GitToken.js | 3 +- forge/ee/lib/gitops/backends/azure.js | 217 ++++++++++ forge/ee/lib/gitops/backends/github.js | 219 ++++++++++ forge/ee/lib/gitops/index.js | 382 +++++++++--------- forge/ee/routes/gitops/index.js | 3 +- frontend/src/components/pipelines/Stage.vue | 4 +- .../pages/application/PipelineStage/form.vue | 8 +- .../src/pages/team/Settings/Integrations.vue | 11 +- .../Settings/dialogs/CreateGitTokenDialog.vue | 17 +- 12 files changed, 688 insertions(+), 199 deletions(-) create mode 100644 forge/db/migrations/20260318-01-EE-extend-gittoken.js create mode 100644 forge/ee/lib/gitops/backends/azure.js create mode 100644 forge/ee/lib/gitops/backends/github.js diff --git a/forge/db/migrations/20260318-01-EE-extend-gittoken.js b/forge/db/migrations/20260318-01-EE-extend-gittoken.js new file mode 100644 index 0000000000..c632ef7a8c --- /dev/null +++ b/forge/db/migrations/20260318-01-EE-extend-gittoken.js @@ -0,0 +1,16 @@ +const { DataTypes } = require('sequelize') + +module.exports = { + /** + * upgrade database + * @param {QueryInterface} context Sequelize.QueryInterface + */ + up: async (context, Sequelize) => { + await context.addColumn('GitTokens', 'type', { + type: DataTypes.STRING, + defaultValue: 'github', + allowNull: false + }) + }, + down: async (context, Sequelize) => { } +} diff --git a/forge/ee/db/models/GitToken.js b/forge/ee/db/models/GitToken.js index 6f39adca26..32cc90fef0 100644 --- a/forge/ee/db/models/GitToken.js +++ b/forge/ee/db/models/GitToken.js @@ -10,6 +10,11 @@ module.exports = { token: { type: DataTypes.STRING, allowNull: false + }, + type: { + type: DataTypes.STRING, + allowNull: false, + default: 'github' } }, associations: function (M) { diff --git a/forge/ee/db/models/PipelineStageGitRepo.js b/forge/ee/db/models/PipelineStageGitRepo.js index f22690cd5b..21453fc974 100644 --- a/forge/ee/db/models/PipelineStageGitRepo.js +++ b/forge/ee/db/models/PipelineStageGitRepo.js @@ -102,6 +102,7 @@ module.exports = { } await app.gitops.pushToRepository({ token: gitToken.token, + tokenType: gitToken.type, url: this.url, branch: this.branch, credentialSecret: this.credentialSecret, @@ -154,6 +155,7 @@ module.exports = { } const snapshotContent = await app.gitops.pullFromRepository({ token: gitToken.token, + tokenType: gitToken.type, url: this.url, branch: this.pullBranch || this.branch, credentialSecret: this.credentialSecret, diff --git a/forge/ee/db/views/GitToken.js b/forge/ee/db/views/GitToken.js index c94b59d7db..026e8752aa 100644 --- a/forge/ee/db/views/GitToken.js +++ b/forge/ee/db/views/GitToken.js @@ -3,7 +3,8 @@ module.exports = { const result = token.toJSON() const filtered = { id: result.hashid, - name: result.name + name: result.name, + type: result.type // Do not include the token value in the response } return filtered diff --git a/forge/ee/lib/gitops/backends/azure.js b/forge/ee/lib/gitops/backends/azure.js new file mode 100644 index 0000000000..807590e03f --- /dev/null +++ b/forge/ee/lib/gitops/backends/azure.js @@ -0,0 +1,217 @@ +const { exec } = require('node:child_process') +const { existsSync } = require('node:fs') +const fs = require('node:fs/promises') +const os = require('node:os') +const path = require('node:path') +const { promisify } = require('node:util') +const execPromised = promisify(exec) + +// const axios = require('axios') + +const { encryptValue, decryptValue } = require('../../../../db/utils') + +async function cloneRepository (url, branch, workingDir) { + try { + await execPromised(`git clone -b ${branch} --depth 1 --single-branch ${url.toString()} .`, { cwd: workingDir }) + } catch (err) { + const output = err.stdout + err.stderr + // Token does not have access to clone the repo + if (/unable to access/.test(output)) { + const result = new Error('Permission denied') + result.code = 'invalid_token' + result.cause = err + throw result + } + // Remote branch does not exist + if (/Could not find remote branch|Remote branch .+ not found/.test(output)) { + const result = new Error('Branch not found') + result.code = 'invalid_branch' + throw result + } + let error + // Fallback - try to extract the 'fatal' line from the output + const m = /fatal: (.*)/.exec(output) + if (m) { + error = new Error('Failed to clone repository: ' + m[1]) + } else { + error = new Error('Failed to clone repository') + } + error.cause = err + throw error + } +} + +module.exports.init = async function (app) { + /** + * Push a snapshot to a git repository + * @param {Object} repoOptions + * @param {String} repoOptions.token + * @param {String} repoOptions.url + * @param {String} repoOptions.branch + * @param {Object} snapshot + * @param {Object} options + * @param {Object} options.sourceObject what produced the snapshot + * @param {Object} options.user who triggered the pipeline + * @param {Object} options.pipeline details of the pipeline + */ + async function pushToRepository (repoOptions, snapshot, options) { + let workingDir + try { + const token = repoOptions.token + const branch = repoOptions.branch || 'main' + if (!/^https:\/\/dev.azure.com/i.test(repoOptions.url)) { + throw new Error('Only Azure repositories are supported') + } + const url = new URL(repoOptions.url) + url.password = token + + // 2. get user details so we can properly attribute the commit + // let userDetails + // try { + // userDetails = await axios.get('https://api.github.com/user', { + // headers: { + // Accept: 'application/vnd.github+json', + // Authorization: `Bearer ${token}`, + // 'X-GitHub-Api-Version': '2022-11-28' + // } + // }) + // } catch (err) { + // const result = new Error('Invalid git token') + // result.code = 'invalid_token' + // result.cause = err + // throw result + // } + + const userGitName = 'flowfuse' // userDetails.data.login + const userGitEmail = 'flowfuse@example.com' // `${userDetails.data.id}+${userDetails.data.login}@users.noreply.github.com` + const author = `${userGitName} <${userGitEmail}>`.replace(/"/g, '\\"') + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + // 3. clone repo + await cloneRepository(url, branch, workingDir) + + // 4. set username/email + await execPromised('git config user.email "no-reply@flowfuse.com"', { cwd: workingDir }) + await execPromised('git config user.name "FlowFuse"', { cwd: workingDir }) + // For local dev - disable gpg signing in case its set in global config + await execPromised('git config commit.gpgsign false', { cwd: workingDir }) + + // 5. export snapshot + const exportOptions = { + credentialSecret: repoOptions.credentialSecret, + components: { + flows: true, + credentials: true + } + } + const result = await app.db.controllers.Snapshot.exportSnapshot(snapshot, exportOptions) + const snapshotExport = app.db.views.ProjectSnapshot.snapshotExport(result) + if (snapshotExport.settings?.settings?.palette?.npmrc) { + const enc = encryptValue(repoOptions.credentialSecret, snapshotExport.settings.settings?.palette?.npmrc) + snapshotExport.settings.settings.palette.npmrc = { $: enc } + } + const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + await fs.writeFile(snapshotFile, JSON.stringify(snapshotExport, null, 4)) + + // 6. stage file + await execPromised(`git add "${snapshotFile}"`, { cwd: workingDir }) + + // 7. commit + await execPromised(`git commit -m "Update snapshot\n\nSnapshot updated by FlowFuse Pipeline '${options.pipeline.name.replace(/"/g, '')}', triggered by ${options.user.username.replace(/"/g, '')}" --author="${author}"`, { cwd: workingDir }) + + try { + // 8. push + await execPromised('git push', { cwd: workingDir }) + } catch (err) { + const output = err.stdout + err.stderr + if (/unable to access/.test(output)) { + const result = new Error('Permission denied') + result.code = 'invalid_token' + result.cause = err + throw result + } + let error + const m = /fatal: (.*)/.exec(output) + if (m) { + error = new Error('Failed to push repository: ' + m[1]) + } else { + error = Error('Failed to push repository') + } + error.cause = err + throw error + } + } finally { + if (workingDir) { + try { + await fs.rm(workingDir, { recursive: true, force: true }) + } catch (err) {} + } + } + } + + /** + * Push a snapshot to a git repository + * @param {Object} repoOptions + * @param {String} repoOptions.token + * @param {String} repoOptions.url + * @param {String} repoOptions.branch + */ + async function pullFromRepository (repoOptions) { + let workingDir + try { + const token = repoOptions.token + const branch = repoOptions.branch || 'main' + if (!/^https:\/\/dev.azure.com/i.test(repoOptions.url)) { + throw new Error('Only Azure repositories are supported') + } + const url = new URL(repoOptions.url) + url.password = token + + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + // 3. clone repo + await cloneRepository(url, branch, workingDir) + + const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + + if (!existsSync(snapshotFile)) { + throw new Error('Snapshot file not found in repository') + } + + try { + const snapshotContent = await fs.readFile(snapshotFile, 'utf8') + const snapshot = JSON.parse(snapshotContent) + if (snapshot.settings?.env) { + const keys = Object.keys(snapshot.settings.env) + keys.forEach((key) => { + const env = snapshot.settings.env[key] + if (env.hidden && env.$) { + // Decrypt the value if it is encrypted + env.value = decryptValue(repoOptions.credentialSecret, env.$) + delete env.$ + } + }) + } + if (snapshot.settings?.settings?.palette?.npmrc) { + const npmrc = snapshot.settings.settings.palette.npmrc + if (typeof npmrc === 'object' && npmrc.$) { + snapshot.settings.settings.palette.npmrc = decryptValue(repoOptions.credentialSecret, npmrc.$) + } + } + return snapshot + } catch (err) { + throw new Error('Failed to read snapshot file: ' + err.message) + } + } finally { + if (workingDir) { + try { + await fs.rm(workingDir, { recursive: true, force: true }) + } catch (err) {} + } + } + } + return { + pushToRepository, + pullFromRepository + } +} diff --git a/forge/ee/lib/gitops/backends/github.js b/forge/ee/lib/gitops/backends/github.js new file mode 100644 index 0000000000..69b533387f --- /dev/null +++ b/forge/ee/lib/gitops/backends/github.js @@ -0,0 +1,219 @@ +const { exec } = require('node:child_process') +const { existsSync } = require('node:fs') +const fs = require('node:fs/promises') +const os = require('node:os') +const path = require('node:path') +const { promisify } = require('node:util') +const execPromised = promisify(exec) + +const axios = require('axios') + +const { encryptValue, decryptValue } = require('../../../../db/utils') + +async function cloneRepository (url, branch, workingDir) { + try { + await execPromised(`git clone -b ${branch} --depth 1 --single-branch ${url.toString()} .`, { cwd: workingDir }) + } catch (err) { + const output = err.stdout + err.stderr + // Token does not have access to clone the repo + if (/unable to access/.test(output)) { + const result = new Error('Permission denied') + result.code = 'invalid_token' + result.cause = err + throw result + } + // Remote branch does not exist + if (/Could not find remote branch|Remote branch .+ not found/.test(output)) { + const result = new Error('Branch not found') + result.code = 'invalid_branch' + throw result + } + let error + // Fallback - try to extract the 'fatal' line from the output + const m = /fatal: (.*)/.exec(output) + if (m) { + error = new Error('Failed to clone repository: ' + m[1]) + } else { + error = new Error('Failed to clone repository') + } + error.cause = err + throw error + } +} + +module.exports.init = async function (app) { + /** + * Push a snapshot to a git repository + * @param {Object} repoOptions + * @param {String} repoOptions.token + * @param {String} repoOptions.url + * @param {String} repoOptions.branch + * @param {Object} snapshot + * @param {Object} options + * @param {Object} options.sourceObject what produced the snapshot + * @param {Object} options.user who triggered the pipeline + * @param {Object} options.pipeline details of the pipeline + */ + async function pushToRepository (repoOptions, snapshot, options) { + let workingDir + try { + const token = repoOptions.token + const branch = repoOptions.branch || 'main' + if (!/^https:\/\/github.com/i.test(repoOptions.url)) { + throw new Error('Only GitHub repositories are supported') + } + const url = new URL(repoOptions.url) + url.username = 'x-access-token' + url.password = token + + // 2. get user details so we can properly attribute the commit + let userDetails + try { + userDetails = await axios.get('https://api.github.com/user', { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + } catch (err) { + const result = new Error('Invalid git token') + result.code = 'invalid_token' + result.cause = err + throw result + } + + const userGitName = userDetails.data.login + const userGitEmail = `${userDetails.data.id}+${userDetails.data.login}@users.noreply.github.com` + const author = `${userGitName} <${userGitEmail}>`.replace(/"/g, '\\"') + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + // 3. clone repo + await cloneRepository(url, branch, workingDir) + + // 4. set username/email + await execPromised('git config user.email "no-reply@flowfuse.com"', { cwd: workingDir }) + await execPromised('git config user.name "FlowFuse"', { cwd: workingDir }) + // For local dev - disable gpg signing in case its set in global config + await execPromised('git config commit.gpgsign false', { cwd: workingDir }) + + // 5. export snapshot + const exportOptions = { + credentialSecret: repoOptions.credentialSecret, + components: { + flows: true, + credentials: true + } + } + const result = await app.db.controllers.Snapshot.exportSnapshot(snapshot, exportOptions) + const snapshotExport = app.db.views.ProjectSnapshot.snapshotExport(result) + if (snapshotExport.settings?.settings?.palette?.npmrc) { + const enc = encryptValue(repoOptions.credentialSecret, snapshotExport.settings.settings?.palette?.npmrc) + snapshotExport.settings.settings.palette.npmrc = { $: enc } + } + const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + await fs.writeFile(snapshotFile, JSON.stringify(snapshotExport, null, 4)) + + // 6. stage file + await execPromised(`git add "${snapshotFile}"`, { cwd: workingDir }) + + // 7. commit + await execPromised(`git commit -m "Update snapshot\n\nSnapshot updated by FlowFuse Pipeline '${options.pipeline.name.replace(/"/g, '')}', triggered by ${options.user.username.replace(/"/g, '')}" --author="${author}"`, { cwd: workingDir }) + + try { + // 8. push + await execPromised('git push', { cwd: workingDir }) + } catch (err) { + const output = err.stdout + err.stderr + if (/unable to access/.test(output)) { + const result = new Error('Permission denied') + result.code = 'invalid_token' + result.cause = err + throw result + } + let error + const m = /fatal: (.*)/.exec(output) + if (m) { + error = new Error('Failed to push repository: ' + m[1]) + } else { + error = Error('Failed to push repository') + } + error.cause = err + throw error + } + } finally { + if (workingDir) { + try { + await fs.rm(workingDir, { recursive: true, force: true }) + } catch (err) {} + } + } + } + + /** + * Push a snapshot to a git repository + * @param {Object} repoOptions + * @param {String} repoOptions.token + * @param {String} repoOptions.url + * @param {String} repoOptions.branch + */ + async function pullFromRepository (repoOptions) { + let workingDir + try { + const token = repoOptions.token + const branch = repoOptions.branch || 'main' + if (!/^https:\/\/github.com/i.test(repoOptions.url)) { + throw new Error('Only GitHub repositories are supported') + } + const url = new URL(repoOptions.url) + url.username = 'x-access-token' + url.password = token + + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + // 3. clone repo + await cloneRepository(url, branch, workingDir) + + const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + + if (!existsSync(snapshotFile)) { + throw new Error('Snapshot file not found in repository') + } + + try { + const snapshotContent = await fs.readFile(snapshotFile, 'utf8') + const snapshot = JSON.parse(snapshotContent) + if (snapshot.settings?.env) { + const keys = Object.keys(snapshot.settings.env) + keys.forEach((key) => { + const env = snapshot.settings.env[key] + if (env.hidden && env.$) { + // Decrypt the value if it is encrypted + env.value = decryptValue(repoOptions.credentialSecret, env.$) + delete env.$ + } + }) + } + if (snapshot.settings?.settings?.palette?.npmrc) { + const npmrc = snapshot.settings.settings.palette.npmrc + if (typeof npmrc === 'object' && npmrc.$) { + snapshot.settings.settings.palette.npmrc = decryptValue(repoOptions.credentialSecret, npmrc.$) + } + } + return snapshot + } catch (err) { + throw new Error('Failed to read snapshot file: ' + err.message) + } + } finally { + if (workingDir) { + try { + await fs.rm(workingDir, { recursive: true, force: true }) + } catch (err) {} + } + } + } + return { + pushToRepository, + pullFromRepository + } +} diff --git a/forge/ee/lib/gitops/index.js b/forge/ee/lib/gitops/index.js index b1d4602202..dbfe71f88d 100644 --- a/forge/ee/lib/gitops/index.js +++ b/forge/ee/lib/gitops/index.js @@ -1,45 +1,45 @@ const { exec } = require('node:child_process') -const { existsSync } = require('node:fs') -const fs = require('node:fs/promises') -const os = require('node:os') -const path = require('node:path') +// const { existsSync } = require('node:fs') +// const fs = require('node:fs/promises') +// const os = require('node:os') +// const path = require('node:path') const { promisify } = require('node:util') const execPromised = promisify(exec) -const axios = require('axios') - -const { encryptValue, decryptValue } = require('../../../db/utils') - -async function cloneRepository (url, branch, workingDir) { - try { - await execPromised(`git clone -b ${branch} --depth 1 --single-branch ${url.toString()} .`, { cwd: workingDir }) - } catch (err) { - const output = err.stdout + err.stderr - // Token does not have access to clone the repo - if (/unable to access/.test(output)) { - const result = new Error('Permission denied') - result.code = 'invalid_token' - result.cause = err - throw result - } - // Remote branch does not exist - if (/Could not find remote branch|Remote branch .+ not found/.test(output)) { - const result = new Error('Branch not found') - result.code = 'invalid_branch' - throw result - } - let error - // Fallback - try to extract the 'fatal' line from the output - const m = /fatal: (.*)/.exec(output) - if (m) { - error = new Error('Failed to clone repository: ' + m[1]) - } else { - error = new Error('Failed to clone repository') - } - error.cause = err - throw error - } -} +// const axios = require('axios') + +// const { encryptValue, decryptValue } = require('../../../db/utils') + +// async function cloneRepository (url, branch, workingDir) { +// try { +// await execPromised(`git clone -b ${branch} --depth 1 --single-branch ${url.toString()} .`, { cwd: workingDir }) +// } catch (err) { +// const output = err.stdout + err.stderr +// // Token does not have access to clone the repo +// if (/unable to access/.test(output)) { +// const result = new Error('Permission denied') +// result.code = 'invalid_token' +// result.cause = err +// throw result +// } +// // Remote branch does not exist +// if (/Could not find remote branch|Remote branch .+ not found/.test(output)) { +// const result = new Error('Branch not found') +// result.code = 'invalid_branch' +// throw result +// } +// let error +// // Fallback - try to extract the 'fatal' line from the output +// const m = /fatal: (.*)/.exec(output) +// if (m) { +// error = new Error('Failed to clone repository: ' + m[1]) +// } else { +// error = new Error('Failed to clone repository') +// } +// error.cause = err +// throw error +// } +// } module.exports.init = async function (app) { // Check if git is installed @@ -51,6 +51,10 @@ module.exports.init = async function (app) { return } + // load backends + const github = await require('./backends/github').init(app) + const azure = await require('./backends/azure').init(app) + // Set the git feature flag app.config.features.register('gitIntegration', true, true) @@ -67,100 +71,107 @@ module.exports.init = async function (app) { * @param {Object} options.pipeline details of the pipeline */ async function pushToRepository (repoOptions, snapshot, options) { - let workingDir - try { - const token = repoOptions.token - const branch = repoOptions.branch || 'main' - if (!/^https:\/\/github.com/i.test(repoOptions.url)) { - throw new Error('Only GitHub repositories are supported') - } - const url = new URL(repoOptions.url) - url.username = 'x-access-token' - url.password = token - - // 2. get user details so we can properly attribute the commit - let userDetails - try { - userDetails = await axios.get('https://api.github.com/user', { - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28' - } - }) - } catch (err) { - const result = new Error('Invalid git token') - result.code = 'invalid_token' - result.cause = err - throw result - } - - const userGitName = userDetails.data.login - const userGitEmail = `${userDetails.data.id}+${userDetails.data.login}@users.noreply.github.com` - const author = `${userGitName} <${userGitEmail}>`.replace(/"/g, '\\"') - workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) - - // 3. clone repo - await cloneRepository(url, branch, workingDir) - - // 4. set username/email - await execPromised('git config user.email "no-reply@flowfuse.com"', { cwd: workingDir }) - await execPromised('git config user.name "FlowFuse"', { cwd: workingDir }) - // For local dev - disable gpg signing in case its set in global config - await execPromised('git config commit.gpgsign false', { cwd: workingDir }) - - // 5. export snapshot - const exportOptions = { - credentialSecret: repoOptions.credentialSecret, - components: { - flows: true, - credentials: true - } - } - const result = await app.db.controllers.Snapshot.exportSnapshot(snapshot, exportOptions) - const snapshotExport = app.db.views.ProjectSnapshot.snapshotExport(result) - if (snapshotExport.settings?.settings?.palette?.npmrc) { - const enc = encryptValue(repoOptions.credentialSecret, snapshotExport.settings.settings?.palette?.npmrc) - snapshotExport.settings.settings.palette.npmrc = { $: enc } - } - const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') - await fs.writeFile(snapshotFile, JSON.stringify(snapshotExport, null, 4)) - - // 6. stage file - await execPromised(`git add "${snapshotFile}"`, { cwd: workingDir }) - - // 7. commit - await execPromised(`git commit -m "Update snapshot\n\nSnapshot updated by FlowFuse Pipeline '${options.pipeline.name.replace(/"/g, '')}', triggered by ${options.user.username.replace(/"/g, '')}" --author="${author}"`, { cwd: workingDir }) - - try { - // 8. push - await execPromised('git push', { cwd: workingDir }) - } catch (err) { - const output = err.stdout + err.stderr - if (/unable to access/.test(output)) { - const result = new Error('Permission denied') - result.code = 'invalid_token' - result.cause = err - throw result - } - let error - const m = /fatal: (.*)/.exec(output) - if (m) { - error = new Error('Failed to push repository: ' + m[1]) - } else { - error = Error('Failed to push repository') - } - error.cause = err - throw error - } - } finally { - if (workingDir) { - try { - await fs.rm(workingDir, { recursive: true, force: true }) - } catch (err) {} - } + if (repoOptions.tokenType === 'github') { + github.pushToRepository(repoOptions, snapshot, options) + } else if (repoOptions.tokenType === 'azure') { + azure.pushToRepository(repoOptions, snapshot, options) } } + // async function pushToRepository (repoOptions, snapshot, options) { + // let workingDir + // try { + // const token = repoOptions.token + // const branch = repoOptions.branch || 'main' + // if (!/^https:\/\/github.com/i.test(repoOptions.url)) { + // throw new Error('Only GitHub repositories are supported') + // } + // const url = new URL(repoOptions.url) + // url.username = 'x-access-token' + // url.password = token + + // // 2. get user details so we can properly attribute the commit + // let userDetails + // try { + // userDetails = await axios.get('https://api.github.com/user', { + // headers: { + // Accept: 'application/vnd.github+json', + // Authorization: `Bearer ${token}`, + // 'X-GitHub-Api-Version': '2022-11-28' + // } + // }) + // } catch (err) { + // const result = new Error('Invalid git token') + // result.code = 'invalid_token' + // result.cause = err + // throw result + // } + + // const userGitName = userDetails.data.login + // const userGitEmail = `${userDetails.data.id}+${userDetails.data.login}@users.noreply.github.com` + // const author = `${userGitName} <${userGitEmail}>`.replace(/"/g, '\\"') + // workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + // // 3. clone repo + // await cloneRepository(url, branch, workingDir) + + // // 4. set username/email + // await execPromised('git config user.email "no-reply@flowfuse.com"', { cwd: workingDir }) + // await execPromised('git config user.name "FlowFuse"', { cwd: workingDir }) + // // For local dev - disable gpg signing in case its set in global config + // await execPromised('git config commit.gpgsign false', { cwd: workingDir }) + + // // 5. export snapshot + // const exportOptions = { + // credentialSecret: repoOptions.credentialSecret, + // components: { + // flows: true, + // credentials: true + // } + // } + // const result = await app.db.controllers.Snapshot.exportSnapshot(snapshot, exportOptions) + // const snapshotExport = app.db.views.ProjectSnapshot.snapshotExport(result) + // if (snapshotExport.settings?.settings?.palette?.npmrc) { + // const enc = encryptValue(repoOptions.credentialSecret, snapshotExport.settings.settings?.palette?.npmrc) + // snapshotExport.settings.settings.palette.npmrc = { $: enc } + // } + // const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + // await fs.writeFile(snapshotFile, JSON.stringify(snapshotExport, null, 4)) + + // // 6. stage file + // await execPromised(`git add "${snapshotFile}"`, { cwd: workingDir }) + + // // 7. commit + // await execPromised(`git commit -m "Update snapshot\n\nSnapshot updated by FlowFuse Pipeline '${options.pipeline.name.replace(/"/g, '')}', triggered by ${options.user.username.replace(/"/g, '')}" --author="${author}"`, { cwd: workingDir }) + + // try { + // // 8. push + // await execPromised('git push', { cwd: workingDir }) + // } catch (err) { + // const output = err.stdout + err.stderr + // if (/unable to access/.test(output)) { + // const result = new Error('Permission denied') + // result.code = 'invalid_token' + // result.cause = err + // throw result + // } + // let error + // const m = /fatal: (.*)/.exec(output) + // if (m) { + // error = new Error('Failed to push repository: ' + m[1]) + // } else { + // error = Error('Failed to push repository') + // } + // error.cause = err + // throw error + // } + // } finally { + // if (workingDir) { + // try { + // await fs.rm(workingDir, { recursive: true, force: true }) + // } catch (err) {} + // } + // } + // } /** * Push a snapshot to a git repository @@ -170,60 +181,67 @@ module.exports.init = async function (app) { * @param {String} repoOptions.branch */ async function pullFromRepository (repoOptions) { - let workingDir - try { - const token = repoOptions.token - const branch = repoOptions.branch || 'main' - if (!/^https:\/\/github.com/i.test(repoOptions.url)) { - throw new Error('Only GitHub repositories are supported') - } - const url = new URL(repoOptions.url) - url.username = 'x-access-token' - url.password = token - - workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) - - // 3. clone repo - await cloneRepository(url, branch, workingDir) - - const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') - - if (!existsSync(snapshotFile)) { - throw new Error('Snapshot file not found in repository') - } - - try { - const snapshotContent = await fs.readFile(snapshotFile, 'utf8') - const snapshot = JSON.parse(snapshotContent) - if (snapshot.settings?.env) { - const keys = Object.keys(snapshot.settings.env) - keys.forEach((key) => { - const env = snapshot.settings.env[key] - if (env.hidden && env.$) { - // Decrypt the value if it is encrypted - env.value = decryptValue(repoOptions.credentialSecret, env.$) - delete env.$ - } - }) - } - if (snapshot.settings?.settings?.palette?.npmrc) { - const npmrc = snapshot.settings.settings.palette.npmrc - if (typeof npmrc === 'object' && npmrc.$) { - snapshot.settings.settings.palette.npmrc = decryptValue(repoOptions.credentialSecret, npmrc.$) - } - } - return snapshot - } catch (err) { - throw new Error('Failed to read snapshot file: ' + err.message) - } - } finally { - if (workingDir) { - try { - await fs.rm(workingDir, { recursive: true, force: true }) - } catch (err) {} - } + if (repoOptions.tokenType === 'github') { + return github.pullFromRepository(repoOptions) + } else if (repoOptions.tokenType === 'azure') { + return azure.pullFromRepository(repoOptions) } } + // async function pullFromRepository (repoOptions) { + // let workingDir + // try { + // const token = repoOptions.token + // const branch = repoOptions.branch || 'main' + // if (!/^https:\/\/github.com/i.test(repoOptions.url)) { + // throw new Error('Only GitHub repositories are supported') + // } + // const url = new URL(repoOptions.url) + // url.username = 'x-access-token' + // url.password = token + + // workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + // // 3. clone repo + // await cloneRepository(url, branch, workingDir) + + // const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + + // if (!existsSync(snapshotFile)) { + // throw new Error('Snapshot file not found in repository') + // } + + // try { + // const snapshotContent = await fs.readFile(snapshotFile, 'utf8') + // const snapshot = JSON.parse(snapshotContent) + // if (snapshot.settings?.env) { + // const keys = Object.keys(snapshot.settings.env) + // keys.forEach((key) => { + // const env = snapshot.settings.env[key] + // if (env.hidden && env.$) { + // // Decrypt the value if it is encrypted + // env.value = decryptValue(repoOptions.credentialSecret, env.$) + // delete env.$ + // } + // }) + // } + // if (snapshot.settings?.settings?.palette?.npmrc) { + // const npmrc = snapshot.settings.settings.palette.npmrc + // if (typeof npmrc === 'object' && npmrc.$) { + // snapshot.settings.settings.palette.npmrc = decryptValue(repoOptions.credentialSecret, npmrc.$) + // } + // } + // return snapshot + // } catch (err) { + // throw new Error('Failed to read snapshot file: ' + err.message) + // } + // } finally { + // if (workingDir) { + // try { + // await fs.rm(workingDir, { recursive: true, force: true }) + // } catch (err) {} + // } + // } + // } return { pushToRepository, pullFromRepository diff --git a/forge/ee/routes/gitops/index.js b/forge/ee/routes/gitops/index.js index bc307d91df..931047384f 100644 --- a/forge/ee/routes/gitops/index.js +++ b/forge/ee/routes/gitops/index.js @@ -48,7 +48,8 @@ module.exports = async function (app) { const token = await app.db.models.GitToken.create({ name: body.name, token: body.token, - TeamId: request.team.id + TeamId: request.team.id, + type: body.type || 'github' }) // TODO: audit log // await app.auditLog.Project.project.httpToken.created(request.session.User, null, request.project, body) diff --git a/frontend/src/components/pipelines/Stage.vue b/frontend/src/components/pipelines/Stage.vue index 2d1d310e80..472220451f 100644 --- a/frontend/src/components/pipelines/Stage.vue +++ b/frontend/src/components/pipelines/Stage.vue @@ -93,8 +93,8 @@
- - {{ stage.gitRepo?.url.replace("https://github.com/","") }} + + {{ stage.gitRepo?.url.replace("https://github.com/","").replace("https://dev.azure.com/","") }}
diff --git a/frontend/src/pages/application/PipelineStage/form.vue b/frontend/src/pages/application/PipelineStage/form.vue index 69e3525c5d..65bfac1814 100644 --- a/frontend/src/pages/application/PipelineStage/form.vue +++ b/frontend/src/pages/application/PipelineStage/form.vue @@ -144,7 +144,7 @@ Choose Git Token - + @@ -19,10 +19,10 @@ Git Integration