From 43faa5b715011fd193a2c53b63accb13f07b0a5d Mon Sep 17 00:00:00 2001 From: Sid Sethi Date: Wed, 19 Aug 2020 15:01:33 -0400 Subject: [PATCH 01/53] Clock work --- .../migrations/20200819145320-vector-clock.js | 121 ++++++++++++++++++ creator-node/src/fileManager.js | 43 +++++-- creator-node/src/models/audiususer.js | 4 + creator-node/src/models/cNodeUser.js | 4 + creator-node/src/models/file.js | 4 + creator-node/src/models/track.js | 4 + creator-node/src/routes/audiusUsers.js | 18 ++- creator-node/src/routes/files.js | 48 ++++--- creator-node/src/routes/tracks.js | 68 ++++++---- creator-node/src/routes/users.js | 27 +++- creator-node/test/lib/dataSeeds.js | 2 +- 11 files changed, 282 insertions(+), 61 deletions(-) create mode 100644 creator-node/sequelize/migrations/20200819145320-vector-clock.js diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200819145320-vector-clock.js new file mode 100644 index 00000000000..bc25dfef70d --- /dev/null +++ b/creator-node/sequelize/migrations/20200819145320-vector-clock.js @@ -0,0 +1,121 @@ +'use strict' + +module.exports = { + up: async (queryInterface, Sequelize) => { + // Add 'clock' column to all 4 data tables + await queryInterface.addColumn('CNodeUsers', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + await queryInterface.addColumn('AudiusUsers', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + await queryInterface.addColumn('Tracks', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + await queryInterface.addColumn('Files', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + + // Add composite uniqueness constraint on (cnodeUserUUID, clock) in each table + await queryInterface.addConstraint( + 'CNodeUsers', + { + type: 'UNIQUE', + fields: ['cnodeUserUUID', 'clock'], + name: 'CNodeUsers_unique_constraint_(cnodeUserUUID,clock)' + } + ) + await queryInterface.addConstraint( + 'AudiusUsers', + { + type: 'UNIQUE', + fields: ['cnodeUserUUID', 'clock'], + name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,clock)' + } + ) + await queryInterface.addConstraint( + 'Tracks', + { + type: 'UNIQUE', + fields: ['cnodeUserUUID', 'clock'], + name: 'Tracks_unique_constraint_(cnodeUserUUID,clock)' + } + ) + + // Add 'isCurrent' column to tables - TBD + // await queryInterface.addColumn('AudiusUsers', 'isCurrent', { + // type: Sequelize.BOOLEAN, + // unique: false, + // allowNull: false + // }) + // await queryInterface.addColumn('Tracks', 'isCurrent', { + // type: Sequelize.BOOLEAN, + // unique: false, + // allowNull: false + // }) + // await queryInterface.addColumn('Files', 'isCurrent', { + // type: Sequelize.BOOLEAN, + // unique: false, + // allowNull: false + // }) + + // TBD - Add constraint on (cnodeUserUUID, isCurrent) in each table to ensure only 1 row marked isCurrent = true + // await queryInterface.addConstraint( + // 'AudiusUsers', + // { + // type: 'UNIQUE', + // fields: ['cnodeUserUUID', 'isCurrent'], + // where: { isCurrent: true }, + // name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,isCurrent)' + // } + // ) + // await queryInterface.addConstraint( + // 'Tracks', + // { + // type: 'UNIQUE', + // fields: ['cnodeUserUUID', 'isCurrent'], + // where: { isCurrent: true }, + // name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,isCurrent)' + // } + // ) + // await queryInterface.addConstraint( + // 'AudiusUsers', + // { + // type: 'UNIQUE', + // fields: ['cnodeUserUUID', 'isCurrent'], + // where: { isCurrent: true }, + // name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,isCurrent)' + // } + // ) + }, + + down: async (queryInterface, Sequelize) => { + // Remove uniqueness constraints on (cnodeUserUUID, clock) on all 4 tables + await queryInterface.removeConstraint( + 'CNodeUsers', + 'CNodeUsers_unique_constraint_(cnodeUserUUID,clock)' + ) + await queryInterface.removeConstraint( + 'AudiusUsers', + 'AudiusUsers_unique_constraint_(cnodeUserUUID,clock)' + ) + await queryInterface.removeConstraint( + 'Tracks', + 'Tracks_unique_constraint_(cnodeUserUUID,clock)' + ) + + // Remove clock columns on all 4 tables + await queryInterface.removeColumn('CNodeUsers', 'clock') + await queryInterface.removeColumn('AudiusUsers', 'clock') + await queryInterface.removeColumn('Tracks', 'clock') + await queryInterface.removeColumn('Files', 'clock') + } +} diff --git a/creator-node/src/fileManager.js b/creator-node/src/fileManager.js index 8105d82be75..933af1e171f 100644 --- a/creator-node/src/fileManager.js +++ b/creator-node/src/fileManager.js @@ -42,14 +42,22 @@ async function saveFileFromBuffer (req, buffer, fileType) { await writeFile(dstPath, buffer) + // fetch highest clock value in CNodeUsers table for cnodeUserUUID + const newClockVal = req.session.cnodeUser.clock + 1 + // add reference to file to database - const file = (await models.File.findOrCreate({ where: { + let file = await models.File.create({ cnodeUserUUID: req.session.cnodeUserUUID, multihash: multihash, sourceFile: req.fileName, storagePath: dstPath, - type: fileType - } }))[0].dataValues + type: fileType, + clock: newClockVal + }) + file = file.dataValues + + // Increment clockVal in cnodeUsers table + await req.session.cnodeUser.update({ clock: newClockVal }) req.logger.info('\nAdded file:', multihash, 'file id', file.fileUUID) return { multihash: multihash, fileUUID: file.fileUUID } @@ -83,18 +91,25 @@ async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, transac req.logger.info(`Time taken in saveFileToIpfsFromFS to copyFileSync: ${Date.now() - codeBlockTimeStart}`) + // fetch highest clock value in CNodeUsers table for cnodeUserUUID + const newClockVal = req.session.cnodeUser.clock + 1 + // add reference to file to database - const queryObj = { where: { - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: multihash, - sourceFile: sourceFile, - storagePath: dstPath, - type: fileType - } } - if (transaction) { - queryObj.transaction = transaction - } - const file = ((await models.File.findOrCreate(queryObj))[0].dataValues) + let file = await models.File.create( + { + cnodeUserUUID: req.session.cnodeUserUUID, + multihash: multihash, + sourceFile: sourceFile, + storagePath: dstPath, + type: fileType, + clock: newClockVal + }, + { transaction } + ) + file = file.dataValues + + // Increment clockVal in cnodeUsers table + await req.session.cnodeUser.update({ clock: newClockVal }, { transaction }) req.logger.info(`Added file: ${multihash} for fileUUID ${file.fileUUID} from sourceFile ${sourceFile}`) return { multihash: multihash, fileUUID: file.fileUUID } diff --git a/creator-node/src/models/audiususer.js b/creator-node/src/models/audiususer.js index 849312dcde5..d5e9c8416c0 100644 --- a/creator-node/src/models/audiususer.js +++ b/creator-node/src/models/audiususer.js @@ -31,6 +31,10 @@ module.exports = (sequelize, DataTypes) => { profilePicFileUUID: { type: DataTypes.UUID, allowNull: true + }, + clock: { + type: DataTypes.INTEGER, + allowNull: false } }, {}) AudiusUser.associate = function (models) { diff --git a/creator-node/src/models/cNodeUser.js b/creator-node/src/models/cNodeUser.js index 277f0c6a3bb..bcdf0ca1bb1 100644 --- a/creator-node/src/models/cNodeUser.js +++ b/creator-node/src/models/cNodeUser.js @@ -21,6 +21,10 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.INTEGER, allowNull: false, defaultValue: -1 + }, + clock: { + type: DataTypes.INTEGER, + allowNull: false } }, {}) diff --git a/creator-node/src/models/file.js b/creator-node/src/models/file.js index a7a5d2d02f9..27dfdb4da0a 100644 --- a/creator-node/src/models/file.js +++ b/creator-node/src/models/file.js @@ -49,6 +49,10 @@ module.exports = (sequelize, DataTypes) => { // track and non types broken down below and attached to Track model isIn: [['track', 'metadata', 'image', 'dir', 'copy320']] } + }, + clock: { + type: DataTypes.INTEGER, + allowNull: false } }, { indexes: [ diff --git a/creator-node/src/models/track.js b/creator-node/src/models/track.js index fd6edba477d..fdab2578d73 100644 --- a/creator-node/src/models/track.js +++ b/creator-node/src/models/track.js @@ -28,6 +28,10 @@ module.exports = (sequelize, DataTypes) => { coverArtFileUUID: { type: DataTypes.UUID, allowNull: true + }, + clock: { + type: DataTypes.INTEGER, + allowNull: false } }, {}) diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index a4c54d26626..697df215a4e 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -6,6 +6,7 @@ const { saveFileFromBuffer } = require('../fileManager') const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers') const { getFileUUIDForImageCID } = require('../utils') const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares') +const { logger } = require('../logging') module.exports = function (app) { /** Create AudiusUser from provided metadata, and make metadata available to network. */ @@ -67,19 +68,26 @@ module.exports = function (app) { } const t = await models.sequelize.transaction() + try { - // Insert / update audiusUser entry on db. - const audiusUser = await models.AudiusUser.upsert({ + logger.info(`beginning audiusUsers DB transactions`) + + // compute new clock value for cnodeUserUUID by incrementing current clock value from CNodeUsers table + const newClockVal = req.session.cnodeUser.clock + 1 + + // Insert new audiusUser entry to DB + const audiusUser = await models.AudiusUser.create({ cnodeUserUUID, metadataFileUUID, metadataJSON, blockchainId: blockchainUserId, coverArtFileUUID, - profilePicFileUUID + profilePicFileUUID, + clock: newClockVal }, { transaction: t, returning: true }) - // Update cnodeUser's latestBlockNumber. - await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction: t }) + // Update cnodeUser's latestBlockNumber and clock + await cnodeUser.update({ latestBlockNumber: blockNumber, clock: newClockVal }, { transaction: t }) await t.commit() triggerSecondarySyncs(req) diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js index ccb286c2295..92723724d94 100644 --- a/creator-node/src/routes/files.js +++ b/creator-node/src/routes/files.js @@ -280,32 +280,44 @@ module.exports = function (app) { const t = await models.sequelize.transaction() // Add the created files to the DB try { + // compute new clock value for cnodeUserUUID by incrementing current clock value from CNodeUsers table + const newClockVal = req.session.cnodeUser.clock + 1 + // Save dir file reference to DB - const dir = (await models.File.findOrCreate({ where: { - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: resizeResp.dir.dirCID, - sourceFile: null, - storagePath: resizeResp.dir.dirDestPath, - type: 'dir' - }, - transaction: t }))[0].dataValues + const dir = (await models.File.create( + { + cnodeUserUUID: req.session.cnodeUserUUID, + multihash: resizeResp.dir.dirCID, + sourceFile: null, + storagePath: resizeResp.dir.dirDestPath, + type: 'dir', + clock: newClockVal + }, + { transaction: t } + )).dataValues // Save each file to the DB await Promise.all(resizeResp.files.map(async (fileResp) => { - const file = (await models.File.findOrCreate({ where: { - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: fileResp.multihash, - sourceFile: fileResp.sourceFile, - storagePath: fileResp.storagePath, - type: 'image', - dirMultihash: resizeResp.dir.dirCID, - fileName: fileResp.sourceFile.split('/').slice(-1)[0] - }, - transaction: t }))[0].dataValues + const file = (await models.File.create( + { + cnodeUserUUID: req.session.cnodeUserUUID, + multihash: fileResp.multihash, + sourceFile: fileResp.sourceFile, + storagePath: fileResp.storagePath, + type: 'image', + dirMultihash: resizeResp.dir.dirCID, + fileName: fileResp.sourceFile.split('/').slice(-1)[0], + clock: newClockVal + }, + { transaction: t } + )).dataValues req.logger.info('Added file', fileResp, file) })) + // Update cnodeUser's clock + await req.session.cnodeUser.update({ clock: newClockVal }, { transaction: t }) + req.logger.info('Added all files for dir', dir) req.logger.info(`route time = ${Date.now() - routestart}`) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 4b87bee2c5c..a421cd6b1de 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -13,6 +13,7 @@ const TranscodingQueue = require('../TranscodingQueue') const { getCID } = require('./files') const { decode } = require('../hashids.js') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') +const { logger } = require('../logging.js') module.exports = function (app) { /** @@ -74,7 +75,7 @@ module.exports = function (app) { response.segmentName = filePath return response })) - transcodedFilePromResp = await saveFileToIPFSFromFS(req, transcodedFilePath, 'copy320', req.fileName) + transcodedFilePromResp = await saveFileToIPFSFromFS(req, transcodedFilePath, 'copy320', req.fileName, t) req.logger.info(`Time taken in /track_content for saving segments and transcoding to IPFS: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) codeBlockTimeStart = Date.now() @@ -157,11 +158,13 @@ module.exports = function (app) { app.post('/tracks/metadata', authMiddleware, ensurePrimaryMiddleware, syncLockMiddleware, handleResponse(async (req, res) => { const metadataJSON = req.body.metadata - if (!metadataJSON || - !metadataJSON.owner_id || - !metadataJSON.track_segments || - !Array.isArray(metadataJSON.track_segments) || - !metadataJSON.track_segments.length) { + if ( + !metadataJSON || + !metadataJSON.owner_id || + !metadataJSON.track_segments || + !Array.isArray(metadataJSON.track_segments) || + !metadataJSON.track_segments.length + ) { return errorResponseBadRequest('Metadata object must include owner_id and non-empty track_segments array') } @@ -204,7 +207,7 @@ module.exports = function (app) { } }) if (!transcodedFile) { - return errorResponseServerError('Failed to find transcoded file ') + return errorResponseServerError('Failed to find transcoded file') } } } @@ -274,25 +277,40 @@ module.exports = function (app) { const t = await models.sequelize.transaction() try { - // Create / update track entry on db. - const resp = (await models.Track.upsert({ - cnodeUserUUID, - metadataFileUUID, - metadataJSON, - blockchainId: blockchainTrackId, - coverArtFileUUID - }, - { transaction: t, returning: true } - )) - const track = resp[0] - const trackCreated = resp[1] - + logger.debug('Beginning POST /tracks DB transactions') + + const existingTrackEntry = await models.Track.findOne({ + where: { + cnodeUserUUID, + metadataFileUUID, + blockchainId: blockchainTrackId, + coverArtFileUUID + }, + transaction: t + }) + + // compute new clock value for cnodeUserUUID by incrementing current clock value from CNodeUsers table + const newClockVal = cnodeUser.clock + 1 + + // Insert new track entry on db (for track update, a new entry is still created with incremented clock val) + const track = await models.Track.create( + { + cnodeUserUUID, + metadataFileUUID, + metadataJSON, + blockchainId: blockchainTrackId, + coverArtFileUUID, + clock: newClockVal + }, + { transaction: t } + ) + /** Associate matching segment files on DB with new/updated track. */ const trackSegmentCIDs = metadataJSON.track_segments.map(segment => segment.multihash) // if track created, ensure files exist with trackuuid = null and update them. - if (trackCreated) { + if (!existingTrackEntry) { // Update the transcoded 320kbps copy if (transcodedTrackUUID) { const transcodedFile = await models.File.findOne({ @@ -394,9 +412,15 @@ module.exports = function (app) { given blockNumber ${blockNumber}` ) if (blockNumber > updatedCNodeUser.latestBlockNumber) { - await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction: t }) + // Update cnodeUser's latestBlockNumber and clock + await cnodeUser.update({ latestBlockNumber: blockNumber, clock: newClockVal }, { transaction: t }) + } else { + // Update cnodeUser's clock + await cnodeUser.update({ clock: newClockVal }, { transaction: t }) } + logger.info(`completed POST tracks route`) + await t.commit() triggerSecondarySyncs(req) return successResponse({ trackUUID: track.trackUUID }) diff --git a/creator-node/src/routes/users.js b/creator-node/src/routes/users.js index 4e0c5f5242a..cdb54bea081 100644 --- a/creator-node/src/routes/users.js +++ b/creator-node/src/routes/users.js @@ -15,6 +15,9 @@ const CHALLENGE_TTL_SECONDS = 120 const CHALLENGE_PREFIX = 'userLoginChallenge:' module.exports = function (app) { + /** + * Creates CNodeUser table entry if one doesn't already exist + */ app.post('/users', handleResponse(async (req, res, next) => { let walletAddress = req.body.walletAddress if (!ethereumUtils.isValidAddress(walletAddress)) { @@ -32,7 +35,11 @@ module.exports = function (app) { return successResponse() // do nothing if user already exists } - await models.CNodeUser.create({ walletPublicKey: walletAddress }) + await models.CNodeUser.create({ + walletPublicKey: walletAddress, + // Initialize clock value for cnodeUser to 0 + clock: 0 + }) return successResponse() })) @@ -145,4 +152,22 @@ module.exports = function (app) { await sessionManager.deleteSession(req.get(sessionManager.sessionTokenHeader)) return successResponse() })) + + app.get('/users/clock_status/:walletPublicKey', handleResponse(async (req, res) => { + let walletPublicKey = req.body.walletPublicKey + + if (!ethereumUtils.isValidAddress(walletPublicKey)) { + return errorResponseBadRequest('Ethereum address is invalid') + } + + walletPublicKey = walletPublicKey.toLowerCase() + + const cnodeUser = await models.CNodeUser.findOne({ + where: { walletPublicKey } + }) + + const clockValue = (cnodeUser) ? cnodeUser.block : -1 + + return successResponse({ clockValue }) + })) } diff --git a/creator-node/test/lib/dataSeeds.js b/creator-node/test/lib/dataSeeds.js index 40a600ee295..73b6960f192 100644 --- a/creator-node/test/lib/dataSeeds.js +++ b/creator-node/test/lib/dataSeeds.js @@ -11,7 +11,7 @@ async function createStarterCNodeUser () { } async function createStarterCNodeUserWithKey (walletPublicKey) { - const cnodeUser = await CNodeUser.create({ walletPublicKey }) + const cnodeUser = await CNodeUser.create({ walletPublicKey, clock: 0 }) return sessionManager.createSession(cnodeUser.cnodeUserUUID) } From 6c9031a64c212228b7488299044bcda7ad2e21d6 Mon Sep 17 00:00:00 2001 From: Sid Sethi Date: Tue, 25 Aug 2020 10:43:45 -0400 Subject: [PATCH 02/53] Fix nodesync + clock_status route --- creator-node/src/routes/nodeSync.js | 17 +++++++++++------ creator-node/src/routes/users.js | 15 ++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 7890a7bcdc9..4c7de9665b9 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -272,11 +272,12 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { req.logger.info(redisKey, `beginning add ops for cnodeUserUUID ${fetchedCnodeUserUUID}`) // Upsert cnodeUser row. - await models.CNodeUser.upsert({ + await models.CNodeUser.create({ cnodeUserUUID: fetchedCnodeUserUUID, walletPublicKey: fetchedWalletPublicKey, latestBlockNumber: fetchedLatestBlockNumber, - lastLogin: fetchedCNodeUser.lastLogin + lastLogin: fetchedCNodeUser.lastLogin, + clock: fetchedCNodeUser.clock }, { transaction: t }) req.logger.info(redisKey, `upserted nodeUser for cnodeUserUUID ${fetchedCnodeUserUUID}`) @@ -329,7 +330,8 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { storagePath: file.storagePath, type: file.type, fileName: file.fileName, - dirMultihash: file.dirMultihash + dirMultihash: file.dirMultihash, + clock: file.clock })), { transaction: t }) req.logger.info(redisKey, 'created all non-track files') @@ -339,7 +341,8 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { cnodeUserUUID: fetchedCnodeUserUUID, metadataJSON: track.metadataJSON, metadataFileUUID: track.metadataFileUUID, - coverArtFileUUID: track.coverArtFileUUID + coverArtFileUUID: track.coverArtFileUUID, + clock: track.clock })), { transaction: t }) req.logger.info(redisKey, 'created all tracks') @@ -353,7 +356,8 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { storagePath: trackFile.storagePath, type: trackFile.type, fileName: trackFile.fileName, - dirMultihash: trackFile.dirMultihash + dirMultihash: trackFile.dirMultihash, + clock: trackFile.clock })), { transaction: t }) req.logger.info('saved all track files to db') @@ -364,7 +368,8 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { metadataJSON: audiusUser.metadataJSON, metadataFileUUID: audiusUser.metadataFileUUID, coverArtFileUUID: audiusUser.coverArtFileUUID, - profilePicFileUUID: audiusUser.profilePicFileUUID + profilePicFileUUID: audiusUser.profilePicFileUUID, + clock: audiusUser.clock })), { transaction: t }) req.logger.info('saved all audiususer data to db') diff --git a/creator-node/src/routes/users.js b/creator-node/src/routes/users.js index cdb54bea081..73afe50e87d 100644 --- a/creator-node/src/routes/users.js +++ b/creator-node/src/routes/users.js @@ -154,19 +154,20 @@ module.exports = function (app) { })) app.get('/users/clock_status/:walletPublicKey', handleResponse(async (req, res) => { - let walletPublicKey = req.body.walletPublicKey + let walletPublicKey = req.params.walletPublicKey - if (!ethereumUtils.isValidAddress(walletPublicKey)) { - return errorResponseBadRequest('Ethereum address is invalid') - } + // TODO - this doesn't work + // if (!ethereumUtils.isValidAddress(walletPublicKey)) { + // return errorResponseBadRequest('Ethereum address is invalid') + // } walletPublicKey = walletPublicKey.toLowerCase() - const cnodeUser = await models.CNodeUser.findOne({ + const cnodeUser = (await models.CNodeUser.findOne({ where: { walletPublicKey } - }) + })).dataValues - const clockValue = (cnodeUser) ? cnodeUser.block : -1 + const clockValue = (cnodeUser) ? cnodeUser.clock : -1 return successResponse({ clockValue }) })) From 151651232ce5ea130e78bfc95868dbe60483b269 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 28 Aug 2020 23:17:50 +0000 Subject: [PATCH 03/53] Fix sync --- creator-node/src/routes/nodeSync.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index f9b1c24a87e..68c1095f00e 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -265,6 +265,12 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { transaction: t }) req.logger.info(redisKey, `numNonTrackFilesDeleted ${numNonTrackFilesDeleted}`) + + // Delete cnodeUser entry + await cnodeUser.destroy({ + transaction: t + }) + req.logger.info(redisKey, `deleted cnodeUserEntry`) } /* Populate all new data for fetched cnodeUser. */ @@ -279,7 +285,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { lastLogin: fetchedCNodeUser.lastLogin, clock: fetchedCNodeUser.clock }, { transaction: t }) - req.logger.info(redisKey, `upserted nodeUser for cnodeUserUUID ${fetchedCnodeUserUUID}`) + req.logger.info(redisKey, `Inserted nodeUser for cnodeUserUUID ${fetchedCnodeUserUUID}`) // Make list of all track Files to add after track creation. From d8768ac37a78403e2f36911768c48e765d94e8f1 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Wed, 2 Sep 2020 04:28:40 +0000 Subject: [PATCH 04/53] Working atomic clock logic with cnodeUser.clock row-level lock --- .../migrations/20200819145320-vector-clock.js | 44 +------------ creator-node/src/fileManager.js | 23 +++---- creator-node/src/routes/audiusUsers.js | 16 ++--- creator-node/src/routes/files.js | 15 +++-- creator-node/src/routes/tracks.js | 61 ++++++++++++------- creator-node/src/routes/users.js | 9 ++- .../utils/incrementAndFetchCNodeUserClock.js | 35 +++++++++++ 7 files changed, 105 insertions(+), 98 deletions(-) create mode 100644 creator-node/src/utils/incrementAndFetchCNodeUserClock.js diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200819145320-vector-clock.js index bc25dfef70d..62c559acc84 100644 --- a/creator-node/sequelize/migrations/20200819145320-vector-clock.js +++ b/creator-node/sequelize/migrations/20200819145320-vector-clock.js @@ -49,50 +49,12 @@ module.exports = { name: 'Tracks_unique_constraint_(cnodeUserUUID,clock)' } ) - - // Add 'isCurrent' column to tables - TBD - // await queryInterface.addColumn('AudiusUsers', 'isCurrent', { - // type: Sequelize.BOOLEAN, - // unique: false, - // allowNull: false - // }) - // await queryInterface.addColumn('Tracks', 'isCurrent', { - // type: Sequelize.BOOLEAN, - // unique: false, - // allowNull: false - // }) - // await queryInterface.addColumn('Files', 'isCurrent', { - // type: Sequelize.BOOLEAN, - // unique: false, - // allowNull: false - // }) - - // TBD - Add constraint on (cnodeUserUUID, isCurrent) in each table to ensure only 1 row marked isCurrent = true - // await queryInterface.addConstraint( - // 'AudiusUsers', - // { - // type: 'UNIQUE', - // fields: ['cnodeUserUUID', 'isCurrent'], - // where: { isCurrent: true }, - // name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,isCurrent)' - // } - // ) - // await queryInterface.addConstraint( - // 'Tracks', - // { - // type: 'UNIQUE', - // fields: ['cnodeUserUUID', 'isCurrent'], - // where: { isCurrent: true }, - // name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,isCurrent)' - // } - // ) // await queryInterface.addConstraint( - // 'AudiusUsers', + // 'Files', // { // type: 'UNIQUE', - // fields: ['cnodeUserUUID', 'isCurrent'], - // where: { isCurrent: true }, - // name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,isCurrent)' + // fields: ['cnodeUserUUID', 'clock'], + // name: 'Files_unique_constraint_(cnodeUserUUID,clock)' // } // ) }, diff --git a/creator-node/src/fileManager.js b/creator-node/src/fileManager.js index 933af1e171f..32873faefc5 100644 --- a/creator-node/src/fileManager.js +++ b/creator-node/src/fileManager.js @@ -15,6 +15,7 @@ const mkdir = promisify(fs.mkdir) const config = require('./config') const models = require('./models') const Utils = require('./utils') +const { incrementAndFetchCNodeUserClock } = require('./utils/incrementAndFetchCNodeUserClock') const MAX_AUDIO_FILE_SIZE = parseInt(config.get('maxAudioFileSizeBytes')) // Default = 250,000,000 bytes = 250MB const MAX_MEMORY_FILE_SIZE = parseInt(config.get('maxMemoryFileSizeBytes')) // Default = 50,000,000 bytes = 50MB @@ -42,22 +43,18 @@ async function saveFileFromBuffer (req, buffer, fileType) { await writeFile(dstPath, buffer) - // fetch highest clock value in CNodeUsers table for cnodeUserUUID - const newClockVal = req.session.cnodeUser.clock + 1 + // increment and fetch cnodeUser.clock value + const newClockVal = await incrementAndFetchCNodeUserClock(req) // add reference to file to database - let file = await models.File.create({ + const file = (await models.File.create({ cnodeUserUUID: req.session.cnodeUserUUID, multihash: multihash, sourceFile: req.fileName, storagePath: dstPath, type: fileType, clock: newClockVal - }) - file = file.dataValues - - // Increment clockVal in cnodeUsers table - await req.session.cnodeUser.update({ clock: newClockVal }) + })).dataValues req.logger.info('\nAdded file:', multihash, 'file id', file.fileUUID) return { multihash: multihash, fileUUID: file.fileUUID } @@ -69,7 +66,7 @@ async function saveFileFromBuffer (req, buffer, fileType) { * - Re-save file to disk under multihash. * - Save reference to file in DB. */ -async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, transaction = null) { +async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, clockVal, transaction = null) { // make sure user has authenticated before saving file if (!req.session.cnodeUserUUID) { throw new Error('User must be authenticated to save a file') @@ -91,9 +88,6 @@ async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, transac req.logger.info(`Time taken in saveFileToIpfsFromFS to copyFileSync: ${Date.now() - codeBlockTimeStart}`) - // fetch highest clock value in CNodeUsers table for cnodeUserUUID - const newClockVal = req.session.cnodeUser.clock + 1 - // add reference to file to database let file = await models.File.create( { @@ -102,15 +96,12 @@ async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, transac sourceFile: sourceFile, storagePath: dstPath, type: fileType, - clock: newClockVal + clock: clockVal }, { transaction } ) file = file.dataValues - // Increment clockVal in cnodeUsers table - await req.session.cnodeUser.update({ clock: newClockVal }, { transaction }) - req.logger.info(`Added file: ${multihash} for fileUUID ${file.fileUUID} from sourceFile ${sourceFile}`) return { multihash: multihash, fileUUID: file.fileUUID } } diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index 697df215a4e..480972d2304 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -6,6 +6,7 @@ const { saveFileFromBuffer } = require('../fileManager') const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers') const { getFileUUIDForImageCID } = require('../utils') const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares') +const { incrementAndFetchCNodeUserClock } = require('../utils/incrementAndFetchCNodeUserClock') const { logger } = require('../logging') module.exports = function (app) { @@ -67,13 +68,12 @@ module.exports = function (app) { return errorResponseBadRequest(e.message) } - const t = await models.sequelize.transaction() - + const transaction = await models.sequelize.transaction() try { logger.info(`beginning audiusUsers DB transactions`) - // compute new clock value for cnodeUserUUID by incrementing current clock value from CNodeUsers table - const newClockVal = req.session.cnodeUser.clock + 1 + // increment and fetch cnodeUser.clock value + const newClockVal = await incrementAndFetchCNodeUserClock(req) // Insert new audiusUser entry to DB const audiusUser = await models.AudiusUser.create({ @@ -84,16 +84,16 @@ module.exports = function (app) { coverArtFileUUID, profilePicFileUUID, clock: newClockVal - }, { transaction: t, returning: true }) + }, { transaction, returning: true }) // Update cnodeUser's latestBlockNumber and clock - await cnodeUser.update({ latestBlockNumber: blockNumber, clock: newClockVal }, { transaction: t }) + await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction }) - await t.commit() + await transaction.commit() triggerSecondarySyncs(req) return successResponse({ audiusUserUUID: audiusUser.audiusUserUUID }) } catch (e) { - await t.rollback() + await transaction.rollback() return errorResponseServerError(e.message) } })) diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js index d6e044bd632..51dec5e8b84 100644 --- a/creator-node/src/routes/files.js +++ b/creator-node/src/routes/files.js @@ -22,6 +22,7 @@ const { authMiddleware, syncLockMiddleware, triggerSecondarySyncs } = require('. const { getIPFSPeerId, ipfsSingleByteCat, ipfsStat } = require('../utils') const ImageProcessingQueue = require('../ImageProcessingQueue') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') +const { incrementAndFetchCNodeUserClock } = require('../utils/incrementAndFetchCNodeUserClock') /** * Helper method to stream file from file system on creator node @@ -285,8 +286,9 @@ module.exports = function (app) { const t = await models.sequelize.transaction() // Add the created files to the DB try { - // compute new clock value for cnodeUserUUID by incrementing current clock value from CNodeUsers table - const newClockVal = req.session.cnodeUser.clock + 1 + // increment and fetch cnodeUser.clock value + const finalClockVal = await incrementAndFetchCNodeUserClock(req, resizeResp.files.length + 1) + const initialClockVal = finalClockVal - (resizeResp.files.length + 1) // Save dir file reference to DB const dir = (await models.File.create( @@ -296,13 +298,13 @@ module.exports = function (app) { sourceFile: null, storagePath: resizeResp.dir.dirDestPath, type: 'dir', - clock: newClockVal + clock: (initialClockVal + 1) }, { transaction: t } )).dataValues // Save each file to the DB - await Promise.all(resizeResp.files.map(async (fileResp) => { + await Promise.all(resizeResp.files.map(async (fileResp, i) => { const file = (await models.File.create( { cnodeUserUUID: req.session.cnodeUserUUID, @@ -312,7 +314,7 @@ module.exports = function (app) { type: 'image', dirMultihash: resizeResp.dir.dirCID, fileName: fileResp.sourceFile.split('/').slice(-1)[0], - clock: newClockVal + clock: (initialClockVal + 1 + (i + 1)) // increment clock val for each file entry }, { transaction: t } )).dataValues @@ -320,9 +322,6 @@ module.exports = function (app) { req.logger.info('Added file', fileResp, file) })) - // Update cnodeUser's clock - await req.session.cnodeUser.update({ clock: newClockVal }, { transaction: t }) - req.logger.info('Added all files for dir', dir) req.logger.info(`route time = ${Date.now() - routestart}`) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index a421cd6b1de..52ffc1fc091 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -14,6 +14,7 @@ const { getCID } = require('./files') const { decode } = require('../hashids.js') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') const { logger } = require('../logging.js') +const { incrementAndFetchCNodeUserClock } = require('../utils/incrementAndFetchCNodeUserClock') module.exports = function (app) { /** @@ -66,16 +67,37 @@ module.exports = function (app) { let segmentSaveFilePromResps let segmentDurations try { + // increment and fetch cnodeUser.clock value + const finalClockVal = await incrementAndFetchCNodeUserClock(req, segmentFilePaths.length + 1) + const initialClockVal = finalClockVal - (segmentFilePaths.length + 1) + + // Call saveFileToIPFSFromFS for transcode file + transcodedFilePromResp = await saveFileToIPFSFromFS( + req, + transcodedFilePath, + 'copy320', + req.fileName, + (initialClockVal + 1), + t + ) + + // Call saveFileToIPFSFromFS for each track segment file req.logger.info(`segmentFilePaths.length ${segmentFilePaths.length}`) - let counter = 1 - segmentSaveFilePromResps = await Promise.all(segmentFilePaths.map(async filePath => { + segmentSaveFilePromResps = await Promise.all(segmentFilePaths.map(async (filePath, i) => { const absolutePath = path.join(req.fileDir, 'segments', filePath) - req.logger.info(`about to perform saveFileToIPFSFromFS #${counter++}`) - let response = await saveFileToIPFSFromFS(req, absolutePath, 'track', req.fileName, t) + req.logger.info(`about to perform saveFileToIPFSFromFS #${i}`) + let response = await saveFileToIPFSFromFS( + req, + absolutePath, + 'track', + req.fileName, + (initialClockVal + 1 + (i + 1)), + t + ) response.segmentName = filePath return response })) - transcodedFilePromResp = await saveFileToIPFSFromFS(req, transcodedFilePath, 'copy320', req.fileName, t) + req.logger.info(`Time taken in /track_content for saving segments and transcoding to IPFS: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) codeBlockTimeStart = Date.now() @@ -289,20 +311,18 @@ module.exports = function (app) { transaction: t }) - // compute new clock value for cnodeUserUUID by incrementing current clock value from CNodeUsers table - const newClockVal = cnodeUser.clock + 1 + // increment and fetch cnodeUser.clock value + const newClockVal = await incrementAndFetchCNodeUserClock(req) // Insert new track entry on db (for track update, a new entry is still created with incremented clock val) - const track = await models.Track.create( - { - cnodeUserUUID, - metadataFileUUID, - metadataJSON, - blockchainId: blockchainTrackId, - coverArtFileUUID, - clock: newClockVal - }, - { transaction: t } + const track = await models.Track.create({ + cnodeUserUUID, + metadataFileUUID, + metadataJSON, + blockchainId: blockchainTrackId, + coverArtFileUUID, + clock: newClockVal + }, { transaction: t } ) /** Associate matching segment files on DB with new/updated track. */ @@ -412,11 +432,8 @@ module.exports = function (app) { given blockNumber ${blockNumber}` ) if (blockNumber > updatedCNodeUser.latestBlockNumber) { - // Update cnodeUser's latestBlockNumber and clock - await cnodeUser.update({ latestBlockNumber: blockNumber, clock: newClockVal }, { transaction: t }) - } else { - // Update cnodeUser's clock - await cnodeUser.update({ clock: newClockVal }, { transaction: t }) + // Update cnodeUser's latestBlockNumber + await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction: t }) } logger.info(`completed POST tracks route`) diff --git a/creator-node/src/routes/users.js b/creator-node/src/routes/users.js index 73afe50e87d..d4ee8abadd2 100644 --- a/creator-node/src/routes/users.js +++ b/creator-node/src/routes/users.js @@ -153,6 +153,9 @@ module.exports = function (app) { return successResponse() })) + /** + * Returns latest clock value stored in CNodeUsers entry given wallet, or -1 if no entry found + */ app.get('/users/clock_status/:walletPublicKey', handleResponse(async (req, res) => { let walletPublicKey = req.params.walletPublicKey @@ -163,11 +166,11 @@ module.exports = function (app) { walletPublicKey = walletPublicKey.toLowerCase() - const cnodeUser = (await models.CNodeUser.findOne({ + const cnodeUser = await models.CNodeUser.findOne({ where: { walletPublicKey } - })).dataValues + }) - const clockValue = (cnodeUser) ? cnodeUser.clock : -1 + const clockValue = (cnodeUser) ? cnodeUser.dataValues.clock : -1 return successResponse({ clockValue }) })) diff --git a/creator-node/src/utils/incrementAndFetchCNodeUserClock.js b/creator-node/src/utils/incrementAndFetchCNodeUserClock.js new file mode 100644 index 00000000000..261085c800a --- /dev/null +++ b/creator-node/src/utils/incrementAndFetchCNodeUserClock.js @@ -0,0 +1,35 @@ +const models = require('../models') + +// TODO consider adding hook to ensure write op can never set clockVal to anything <= current +const incrementAndFetchCNodeUserClock = async (req, incrementBy = 1) => { + req.logger.error(`SIDTEST - BEGINNING CNODE USER UPDATE OP`) + + let newClockVal + const transaction = await models.sequelize.transaction() + + try { + const cnodeUser = await models.CNodeUser.findOne({ + where: { cnodeUserUUID: req.session.cnodeUserUUID }, + transaction, + lock: transaction.LOCK.UPDATE + }) + newClockVal = cnodeUser.clock + incrementBy + await cnodeUser.update( + { clock: newClockVal }, + { transaction } + ) + + await transaction.commit() + req.logger.error(`SIDTEST - COMPLETED CNODE USER UPDATE OP`) + } catch (e) { + await transaction.rollback() + req.logger.error(`SIDTEST - FAILED CNODE USER UPDATE OP`) + throw new Error('Failed to increment cnodeUser.clock') + } + + return newClockVal +} + +module.exports = { + incrementAndFetchCNodeUserClock +} \ No newline at end of file From 4ef5749cb6d464c860fff2ce514d4c675af9e213 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 3 Sep 2020 14:54:05 +0000 Subject: [PATCH 05/53] Pass tx and clockVal to saveFileFromBuffer() + fix tests --- creator-node/src/fileManager.js | 30 +++++++++---------- creator-node/src/routes/audiusUsers.js | 17 +++++++++-- creator-node/src/routes/tracks.js | 19 ++++++++++-- .../utils/incrementAndFetchCNodeUserClock.js | 4 +-- .../incrementAndFetchCNodeUserClock.test.js | 7 +++++ creator-node/test/fileManager.js | 26 ++++++++-------- 6 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js diff --git a/creator-node/src/fileManager.js b/creator-node/src/fileManager.js index 32873faefc5..7ddc388df25 100644 --- a/creator-node/src/fileManager.js +++ b/creator-node/src/fileManager.js @@ -15,7 +15,6 @@ const mkdir = promisify(fs.mkdir) const config = require('./config') const models = require('./models') const Utils = require('./utils') -const { incrementAndFetchCNodeUserClock } = require('./utils/incrementAndFetchCNodeUserClock') const MAX_AUDIO_FILE_SIZE = parseInt(config.get('maxAudioFileSizeBytes')) // Default = 250,000,000 bytes = 250MB const MAX_MEMORY_FILE_SIZE = parseInt(config.get('maxMemoryFileSizeBytes')) // Default = 50,000,000 bytes = 50MB @@ -29,7 +28,7 @@ const AUDIO_MIME_TYPE_REGEX = /audio\/(.*)/ * @dev - only call this function when file is not already stored to disk * - if it is, then use saveFileToIPFSFromFS() */ -async function saveFileFromBuffer (req, buffer, fileType) { +async function saveFileFromBuffer (req, buffer, fileType, clockVal, transaction = null) { // make sure user has authenticated before saving file if (!req.session.cnodeUserUUID) { throw new Error('User must be authenticated to save a file') @@ -43,18 +42,18 @@ async function saveFileFromBuffer (req, buffer, fileType) { await writeFile(dstPath, buffer) - // increment and fetch cnodeUser.clock value - const newClockVal = await incrementAndFetchCNodeUserClock(req) - // add reference to file to database - const file = (await models.File.create({ - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: multihash, - sourceFile: req.fileName, - storagePath: dstPath, - type: fileType, - clock: newClockVal - })).dataValues + const file = (await models.File.create( + { + cnodeUserUUID: req.session.cnodeUserUUID, + multihash: multihash, + sourceFile: req.fileName, + storagePath: dstPath, + type: fileType, + clock: clockVal + }, + { transaction } + )).dataValues req.logger.info('\nAdded file:', multihash, 'file id', file.fileUUID) return { multihash: multihash, fileUUID: file.fileUUID } @@ -89,7 +88,7 @@ async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, clockVa req.logger.info(`Time taken in saveFileToIpfsFromFS to copyFileSync: ${Date.now() - codeBlockTimeStart}`) // add reference to file to database - let file = await models.File.create( + const file = (await models.File.create( { cnodeUserUUID: req.session.cnodeUserUUID, multihash: multihash, @@ -99,8 +98,7 @@ async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, clockVa clock: clockVal }, { transaction } - ) - file = file.dataValues + )).dataValues req.logger.info(`Added file: ${multihash} for fileUUID ${file.fileUUID} from sourceFile ${sourceFile}`) return { multihash: multihash, fileUUID: file.fileUUID } diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index 480972d2304..de14ec487df 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -16,13 +16,26 @@ module.exports = function (app) { const metadataJSON = req.body.metadata const metadataBuffer = Buffer.from(JSON.stringify(metadataJSON)) - let multihash, fileUUID + const transaction = await models.sequelize.transaction() + let multihash, fileUUID try { - const saveFileFromBufferResp = await saveFileFromBuffer(req, metadataBuffer, 'metadata') + // increment and fetch cnodeUser.clock value + const newClockVal = await incrementAndFetchCNodeUserClock(req) + + const saveFileFromBufferResp = await saveFileFromBuffer( + req, + metadataBuffer, + 'metadata', + newClockVal, + transaction + ) multihash = saveFileFromBufferResp.multihash fileUUID = saveFileFromBufferResp.fileUUID + + await transaction.commit() } catch (e) { + await transaction.rollback() return errorResponseServerError(`Could not save file to disk, ipfs, and/or db: ${e}`) } diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 52ffc1fc091..1eefc633811 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -237,12 +237,25 @@ module.exports = function (app) { // Store + pin metadata multihash to disk + IPFS. const metadataBuffer = Buffer.from(JSON.stringify(metadataJSON)) + const transaction = await models.sequelize.transaction() let multihash, fileUUID try { - const saveFileFromBufferResp = await saveFileFromBuffer(req, metadataBuffer, 'metadata') + // increment and fetch cnodeUser.clock value + const newClockVal = await incrementAndFetchCNodeUserClock(req) + + const saveFileFromBufferResp = await saveFileFromBuffer( + req, + metadataBuffer, + 'metadata', + newClockVal, + transaction + ) multihash = saveFileFromBufferResp.multihash fileUUID = saveFileFromBufferResp.fileUUID + + await transaction.commit() } catch (e) { + await transaction.rollback() return errorResponseServerError(`Could not save file to disk, ipfs, and/or db: ${e}`) } @@ -322,9 +335,9 @@ module.exports = function (app) { blockchainId: blockchainTrackId, coverArtFileUUID, clock: newClockVal - }, { transaction: t } + }, { transaction: t } ) - + /** Associate matching segment files on DB with new/updated track. */ const trackSegmentCIDs = metadataJSON.track_segments.map(segment => segment.multihash) diff --git a/creator-node/src/utils/incrementAndFetchCNodeUserClock.js b/creator-node/src/utils/incrementAndFetchCNodeUserClock.js index 261085c800a..f9e8bf053f4 100644 --- a/creator-node/src/utils/incrementAndFetchCNodeUserClock.js +++ b/creator-node/src/utils/incrementAndFetchCNodeUserClock.js @@ -31,5 +31,5 @@ const incrementAndFetchCNodeUserClock = async (req, incrementBy = 1) => { } module.exports = { - incrementAndFetchCNodeUserClock -} \ No newline at end of file + incrementAndFetchCNodeUserClock +} diff --git a/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js b/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js new file mode 100644 index 00000000000..02283da0faa --- /dev/null +++ b/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js @@ -0,0 +1,7 @@ +// const assert = require('assert') + +// const { incrementAndFetchCNodeUserClock } = require('./incrementAndFetchCNodeUserClock') + +// describe('Test incrementAndFetchCNodeUserClock', () => { + +// }) diff --git a/creator-node/test/fileManager.js b/creator-node/test/fileManager.js index f6eef32a923..5511ab9bf41 100644 --- a/creator-node/test/fileManager.js +++ b/creator-node/test/fileManager.js @@ -24,7 +24,8 @@ const req = { cnodeUserUUID: uuid() }, logger: { - info: () => {} + info: () => {}, + error: () => {} }, app: { get: key => { @@ -47,6 +48,7 @@ const metadata = { owner_id: 1 } const buffer = Buffer.from(JSON.stringify(metadata)) +const clockVal = 1 describe('test fileManager', () => { afterEach(function () { @@ -72,7 +74,7 @@ describe('test fileManager', () => { } try { - await saveFileToIPFSFromFS(reqOverride, srcPath, fileType, sourceFile) + await saveFileToIPFSFromFS(reqOverride, srcPath, fileType, sourceFile, clockVal) assert.fail('Should not have passed if cnodeUserUUID is not present in request.') } catch (e) { assert.deepStrictEqual(e.message, 'User must be authenticated to save a file') @@ -88,7 +90,7 @@ describe('test fileManager', () => { sinon.stub(ipfs, 'addFromFs').rejects(new Error('ipfs is down!')) try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile) + await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) assert.fail('Should not have passed if ipfs is down.') } catch (e) { assert.deepStrictEqual(e.message, 'ipfs is down!') @@ -104,7 +106,7 @@ describe('test fileManager', () => { sinon.stub(fs, 'copyFileSync').throws(new Error('Failed to copy files!!')) try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile) + await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) assert.fail('Should not have passed if file copying fails.') } catch (e) { assert.deepStrictEqual(e.message, 'Failed to copy files!!') @@ -117,10 +119,10 @@ describe('test fileManager', () => { * Then: an error is thrown */ it('should throw an error if db connection is down', async () => { - sinon.stub(models.File, 'findOrCreate').rejects(new Error('Failed to find or create file!!!')) + sinon.stub(models.File, 'create').rejects(new Error('Failed to find or create file!!!')) try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile) + await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) assert.fail('Should not have passed if db connection is down.') } catch (e) { assert.deepStrictEqual(e.message, 'Failed to find or create file!!!') @@ -136,10 +138,10 @@ describe('test fileManager', () => { * - that segment should be present in IPFS */ it('should pass saving file to ipfs from fs (happy path)', async () => { - sinon.stub(models.File, 'findOrCreate').returns([{ dataValues: 'data' }]) + sinon.stub(models.File, 'create').returns({ dataValues: { fileUUID: 'uuid' } }) try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile) + await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) } catch (e) { assert.fail(e.message) } @@ -231,10 +233,10 @@ describe('test fileManager', () => { * Then: an error is thrown */ it('should throw an error if writing reference to db fails', async () => { - sinon.stub(models.File, 'findOrCreate').rejects(new Error('Failed to find or create file!!!')) + sinon.stub(models.File, 'create').rejects(new Error('Failed to find or create file!!!')) try { - await saveFileFromBuffer(req, buffer, 'metadata') + await saveFileFromBuffer(req, buffer, 'metadata', clockVal) assert.fail('Should not have if db connection is down.') } catch (e) { assert.deepStrictEqual(e.message, 'Failed to find or create file!!!') @@ -247,11 +249,11 @@ describe('test fileManager', () => { * Then: ipfs, fs, and db should have the buffer contents */ it('should pass saving file from buffer (happy path)', async () => { - sinon.stub(models.File, 'findOrCreate').returns([{ dataValues: { fileUUID: 'uuid' } }]) + sinon.stub(models.File, 'create').returns({ dataValues: { fileUUID: 'uuid' } }) let resp try { - resp = await saveFileFromBuffer(req, buffer, 'metadata') + resp = await saveFileFromBuffer(req, buffer, 'metadata', clockVal) } catch (e) { assert.fail(e.message) } From 704d51cef375e67bf1ec422560da170f180fc049 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 4 Sep 2020 06:02:17 +0000 Subject: [PATCH 06/53] Add incrementAndFetchCNodeUserClock tests --- creator-node/package-lock.json | 28 +++++- creator-node/package.json | 1 + creator-node/scripts/run-tests.sh | 4 +- creator-node/src/models/index.js | 4 + .../utils/incrementAndFetchCNodeUserClock.js | 13 ++- .../incrementAndFetchCNodeUserClock.test.js | 89 ++++++++++++++++++- creator-node/src/utils/requestRange.test.js | 2 +- creator-node/test/lib/dataSeeds.js | 6 +- 8 files changed, 128 insertions(+), 19 deletions(-) diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json index 2c78eb517de..1d874878aac 100644 --- a/creator-node/package-lock.json +++ b/creator-node/package-lock.json @@ -4068,6 +4068,15 @@ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, + "fill-keys": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", + "integrity": "sha1-mo+jb06K1jTjv2tPPIiCVRRS6yA=", + "requires": { + "is-object": "~1.0.1", + "merge-descriptors": "~1.0.0" + } + }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -7676,6 +7685,11 @@ "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-4.12.0.tgz", "integrity": "sha512-/P/HtrlvBxY4o/PzXY9cCNBrdylDNxg7gnrv2sMNxj+UJ2m8jSpl0/A6fuJeNAWr99ZvGWH8XCbE0vmnM5KupQ==" }, + "module-not-found-error": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz", + "integrity": "sha1-z4tP9PKWQGdNbN0CsOO8UjwrvcA=" + }, "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", @@ -8821,8 +8835,7 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", - "dev": true + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" }, "path-to-regexp": { "version": "0.1.7", @@ -9288,6 +9301,16 @@ "ipaddr.js": "1.9.0" } }, + "proxyquire": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/proxyquire/-/proxyquire-2.1.3.tgz", + "integrity": "sha512-BQWfCqYM+QINd+yawJz23tbBM40VIGXOdDw3X344KcclI/gtBbdWF6SlQ4nK/bYhF9d27KYug9WzljHC6B9Ysg==", + "requires": { + "fill-keys": "^1.0.2", + "module-not-found-error": "^1.0.1", + "resolve": "^1.11.1" + } + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -9714,7 +9737,6 @@ "version": "1.12.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.12.0.tgz", "integrity": "sha512-B/dOmuoAik5bKcD6s6nXDCjzUKnaDvdkRyAk6rsmsKLipWj4797iothd7jmmUhWTfinVMU+wc56rYKsit2Qy4w==", - "dev": true, "requires": { "path-parse": "^1.0.6" } diff --git a/creator-node/package.json b/creator-node/package.json index 592cc90d6cf..993f06636b0 100644 --- a/creator-node/package.json +++ b/creator-node/package.json @@ -47,6 +47,7 @@ "lodash": "^4.17.15", "multer": "^1.4.0", "pg": "^7.6.1", + "proxyquire": "^2.1.3", "rate-limit-redis": "^1.6.0", "sequelize": "^4.41.2", "shortid": "^2.2.14", diff --git a/creator-node/scripts/run-tests.sh b/creator-node/scripts/run-tests.sh index da5a905152f..f9816abe58a 100755 --- a/creator-node/scripts/run-tests.sh +++ b/creator-node/scripts/run-tests.sh @@ -29,7 +29,7 @@ tear_down () { run_unit_tests () { set +e echo Running unit tests... - ./node_modules/mocha/bin/mocha src/**/*.test.js + ./node_modules/mocha/bin/mocha src/**/*.test.js --timeout 100000 --exit set -e } @@ -100,7 +100,7 @@ export delegateOwnerWallet="0x1eC723075E67a1a2B6969dC5CfF0C6793cb36D25" export delegatePrivateKey="0xdb527e4d4a2412a443c17e1666764d3bba43e89e61129a35f9abc337ec170a5d" # tests -run_unit_tests +# run_unit_tests run_integration_tests rm -r $storagePath diff --git a/creator-node/src/models/index.js b/creator-node/src/models/index.js index b7b3c95d569..de1c83b66e0 100644 --- a/creator-node/src/models/index.js +++ b/creator-node/src/models/index.js @@ -12,6 +12,10 @@ const db = {} const sequelize = new Sequelize(globalConfig.get('dbUrl'), { logging: true, operatorsAliases: false, + // dialectOptions: { + // idleTimeoutMillis: 500, + // connectionTimeoutMillis: 500 + // }, pool: { max: 100, min: 5, diff --git a/creator-node/src/utils/incrementAndFetchCNodeUserClock.js b/creator-node/src/utils/incrementAndFetchCNodeUserClock.js index f9e8bf053f4..366be675fa5 100644 --- a/creator-node/src/utils/incrementAndFetchCNodeUserClock.js +++ b/creator-node/src/utils/incrementAndFetchCNodeUserClock.js @@ -2,32 +2,29 @@ const models = require('../models') // TODO consider adding hook to ensure write op can never set clockVal to anything <= current const incrementAndFetchCNodeUserClock = async (req, incrementBy = 1) => { - req.logger.error(`SIDTEST - BEGINNING CNODE USER UPDATE OP`) - - let newClockVal const transaction = await models.sequelize.transaction() try { const cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID }, transaction, + /** TODO add comment */ lock: transaction.LOCK.UPDATE }) - newClockVal = cnodeUser.clock + incrementBy + + const newClockVal = (cnodeUser.clock + incrementBy) + await cnodeUser.update( { clock: newClockVal }, { transaction } ) await transaction.commit() - req.logger.error(`SIDTEST - COMPLETED CNODE USER UPDATE OP`) + return newClockVal } catch (e) { await transaction.rollback() - req.logger.error(`SIDTEST - FAILED CNODE USER UPDATE OP`) throw new Error('Failed to increment cnodeUser.clock') } - - return newClockVal } module.exports = { diff --git a/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js b/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js index 02283da0faa..66b09e7af49 100644 --- a/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js +++ b/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js @@ -1,7 +1,88 @@ -// const assert = require('assert') +const assert = require('assert') +const proxyquire = require('proxyquire') +const _ = require('lodash') -// const { incrementAndFetchCNodeUserClock } = require('./incrementAndFetchCNodeUserClock') +const models = require('../models') +const { createStarterCNodeUser } = require('../../test/lib/dataSeeds') +const { incrementAndFetchCNodeUserClock } = require('./incrementAndFetchCNodeUserClock') +const utils = require('../utils') -// describe('Test incrementAndFetchCNodeUserClock', () => { +describe('Test incrementAndFetchCNodeUserClock', () => { + const req = { + logger: { + error: (msg) => console.log(msg) + } + } -// }) + const initialClockVal = 0 + const incrementClockBy = 1 + + /** Create cnodeUser */ + beforeEach(async () => { + const resp = await createStarterCNodeUser() + req.session = { cnodeUserUUID: resp.cnodeUserUUID } + + // Confirm initial clock val in DB + let cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) + assert.strictEqual(cnodeUser.clock, initialClockVal) + }) + + /** Wipe all CNodeUsers + dependent data */ + afterEach(async () => { + await models.CNodeUser.destroy({ + where: {}, + truncate: true, + cascade: true // cascades delete to all rows with foreign key on cnodeUser + }) + }) + + it('Sequential increment&Fetch', async () => { + // Explicitly pass in incrementBy value + const newClockVal1 = await incrementAndFetchCNodeUserClock(req, incrementClockBy) + + // Confirm function response + assert.strictEqual(newClockVal1, initialClockVal + incrementClockBy) + + // Confirm clock val in DB + cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) + assert.strictEqual(cnodeUser.clock, initialClockVal + incrementClockBy) + + // Use default incrementBy value + const newClockVal2 = await incrementAndFetchCNodeUserClock(req) + + // Confirm function response + assert.strictEqual(newClockVal2, newClockVal1 + incrementClockBy) + + // Confirm clock val in DB + cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) + assert.strictEqual(cnodeUser.clock, newClockVal1 + incrementClockBy) + }) + + it.only('Concurrent increment&Fetch', async () => { + // Add global sequelize hook to add timeout before cnodeUser.update calls to force concurrent transactions + const modelsCopy = models + modelsCopy.sequelize.addHook('beforeUpdate', async (instance, options) => { + if (instance.constructor.name === 'CNodeUser') { + await utils.timeout(5000) + } + }) + + // Replace required models instance with modified models instance + proxyquire('./incrementAndFetchCNodeUserClock.js', { '../models': modelsCopy }) + + // Fire 10 increment&Fetch calls in parallel + const arr = _.range(1,11) // [1,2,3,4,5,6,7,8,9,10] + const returnedClockVals = await Promise.all(arr.map(async (i) => { + console.log(`calling increment and fetch ${i}...`) + return incrementAndFetchCNodeUserClock(req) + })) + + // Ensure returned clock values include no duplicates and include each value from 1-10, in any order + const returnedClockValsSorted = returnedClockVals.sort((a, b) => a - b) + assert.deepEqual(returnedClockValsSorted, arr) + }) + + it('Force blocked requests to fail', async () => { + + }) +}) diff --git a/creator-node/src/utils/requestRange.test.js b/creator-node/src/utils/requestRange.test.js index 7dd42bf7449..d1a81798c29 100644 --- a/creator-node/src/utils/requestRange.test.js +++ b/creator-node/src/utils/requestRange.test.js @@ -45,6 +45,6 @@ describe('Test formatContentRange', () => { it('Should use size when end is unset', () => { const header = formatContentRange(1024, undefined, 4096) - assert.strictEqual(header, 'bytes 1024-4096/4096') + assert.strictEqual(header, 'bytes 1024-4095/4096') }) }) diff --git a/creator-node/test/lib/dataSeeds.js b/creator-node/test/lib/dataSeeds.js index 73b6960f192..0a7f9c11519 100644 --- a/creator-node/test/lib/dataSeeds.js +++ b/creator-node/test/lib/dataSeeds.js @@ -12,7 +12,11 @@ async function createStarterCNodeUser () { async function createStarterCNodeUserWithKey (walletPublicKey) { const cnodeUser = await CNodeUser.create({ walletPublicKey, clock: 0 }) - return sessionManager.createSession(cnodeUser.cnodeUserUUID) + const sessionToken = await sessionManager.createSession(cnodeUser.cnodeUserUUID) + return { + cnodeUserUUID: cnodeUser.cnodeUserUUID, + sessionToken: sessionToken + } } module.exports = { createStarterCNodeUser, createStarterCNodeUserWithKey, testEthereumConstants } From cc84a67e5573312e871439a21f76c8b12295c6c4 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 4 Sep 2020 07:45:08 +0000 Subject: [PATCH 07/53] WIP data migration --- creator-node/scripts/clock-data-migration.js | 64 +++++++++++++++++++ creator-node/scripts/dev-server.sh | 2 +- creator-node/scripts/discprov-users.txt | 10 +++ .../migrations/20200819145320-vector-clock.js | 23 +++++++ creator-node/src/config.js | 8 +-- 5 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 creator-node/scripts/clock-data-migration.js create mode 100644 creator-node/scripts/discprov-users.txt diff --git a/creator-node/scripts/clock-data-migration.js b/creator-node/scripts/clock-data-migration.js new file mode 100644 index 00000000000..aeef9d996e7 --- /dev/null +++ b/creator-node/scripts/clock-data-migration.js @@ -0,0 +1,64 @@ +const fs = require('fs-extra') +const assert = require('assert') + +const models = require('../src/models') + + +const discprovUsersFilePath = './discprov-users.txt' + +const usersToRSetMap = {} +const nodeToUsersMap = {} + +const buildNodeToUsersMap = async () => { + const fileData = fs.readFileSync(discprovUsersFilePath, 'utf8') + + const fileDataLines = fileData.split('\n') + + for (const fileDataLine of fileDataLines) { + const [userId, wallet, endpointStr] = fileDataLine.split('\t') + // console.log(userId, endpointStr) + + usersToRSetMap[userId] = { wallet, endpointStr } + + const [primary, ...secondaries] = endpointStr.split(',') + if (nodeToUsersMap[primary]) { + nodeToUsersMap[primary].push({ userId, secondaries }) + } else { + nodeToUsersMap[primary] = [{ userId, secondaries }] + } + } + + // console.log('\n\n') + // console.log(JSON.stringify(nodeToUsersMap, undefined, 2)) + + assert.equal(fileDataLines.length, (Object.keys(usersToRSetMap)).length) +} + +/** +for (primary of primaries) + for (user_id of user_ids) + update clock state on primary (in transaction) + select cnodeUserUUID from AudiusUsers for blockchainId + select all rows in Files, Tracks, AudiusUsers for cnodeUserUUID + order all data in time order asc + assign auto-inc clcokval to each row from 1 + assign final clockval to cnodeUsers row + force sync secondaries against primary + */ +const populateClockVals = async () => { + const nodes = Object.keys(nodeToUsersMap) + + for (const node of nodes) { + for (const { userId, secondaries } of nodeToUsersMap[node]) { + // console.log(node, userId, secondaries) + // const { userId, secondaries } = userObj + + // const transaction = await models.sequelize.transaction() + const resp = await models.CNodeUser.find() + console.log(resp) + } + } +} + +buildNodeToUsersMap() +populateClockVals() diff --git a/creator-node/scripts/dev-server.sh b/creator-node/scripts/dev-server.sh index e6df96fa2fa..d02653d2994 100755 --- a/creator-node/scripts/dev-server.sh +++ b/creator-node/scripts/dev-server.sh @@ -2,7 +2,7 @@ set -o xtrace set -e -link_libs=false +link_libs=true if [ "$link_libs" = true ] then diff --git a/creator-node/scripts/discprov-users.txt b/creator-node/scripts/discprov-users.txt new file mode 100644 index 00000000000..77abb86e656 --- /dev/null +++ b/creator-node/scripts/discprov-users.txt @@ -0,0 +1,10 @@ +1 0x17ad89ebc6f0e12e963288e60ee81a1e6e181b91 http://cn1_creator-node_1:4000,http://cn3_creator-node_1:4002,http://cn2_creator-node_1:4001 +2 0xbbc0c90c33d865cc577d261cad7a6ab9b3fc8997 http://cn1_creator-node_1:4000,http://cn3_creator-node_1:4002,http://cn2_creator-node_1:4001 +3 0xcb4a77e905c14eb7a327b3fdc22b82f4e71031ef http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 +4 0x8e70d3452d9575f4acdc11fe6d8ebdc100604f0a http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 +5 0x9a5a9e5c80f5045616e6d7962dcac98e94f29147 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 +6 0x9c690258d041d3d35760fe702e9921d73c0cd61f http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 +7 0x8801eba57efad532db64907b182071c79712b5a2 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 +8 0xe5982822f5a4a1c1be67c35762cf56508940b674 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 +9 0x1f491b220e45d2f9cea88a65977c23813d213833 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 +10 0xf62736cf02e457db2f24f71cb94ad6a86a64036e http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 \ No newline at end of file diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200819145320-vector-clock.js index 62c559acc84..98fb472cc19 100644 --- a/creator-node/sequelize/migrations/20200819145320-vector-clock.js +++ b/creator-node/sequelize/migrations/20200819145320-vector-clock.js @@ -2,6 +2,29 @@ module.exports = { up: async (queryInterface, Sequelize) => { + // Add 'clock2' column to all 4 data tables - TESTING ONLY + await queryInterface.addColumn('CNodeUsers', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + await queryInterface.addColumn('AudiusUsers', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + await queryInterface.addColumn('Tracks', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + await queryInterface.addColumn('Files', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull: false + }) + + // Add 'clock' column to all 4 data tables await queryInterface.addColumn('CNodeUsers', 'clock', { type: Sequelize.INTEGER, diff --git a/creator-node/src/config.js b/creator-node/src/config.js index c403add28aa..5d4382d2733 100644 --- a/creator-node/src/config.js +++ b/creator-node/src/config.js @@ -386,10 +386,10 @@ const config = convict({ */ // TODO(DM) - remove these defaults -const defaultConfigExists = fs.existsSync('default-config.json') -if (defaultConfigExists) config.loadFile('default-config.json') +const defaultConfigExists = fs.existsSync('../default-config.json') +if (defaultConfigExists) config.loadFile('../default-config.json') -if (fs.existsSync('eth-contract-config.json')) { +if (fs.existsSync('../eth-contract-config.json')) { let ethContractConfig = require('../eth-contract-config.json') config.load({ 'ethTokenAddress': ethContractConfig.audiusTokenAddress, @@ -399,7 +399,7 @@ if (fs.existsSync('eth-contract-config.json')) { }) } -if (fs.existsSync('contract-config.json')) { +if (fs.existsSync('../contract-config.json')) { const dataContractConfig = require('../contract-config.json') config.load({ 'dataRegistryAddress': dataContractConfig.registryAddress From 66768a46c69df3f82722a4e7f86b04d5d862d920 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 4 Sep 2020 21:52:24 +0000 Subject: [PATCH 08/53] More migration work --- creator-node/scripts/clock-data-migration.js | 29 ++++++++--- creator-node/scripts/db.js | 51 +++++++++++++++++++ .../migrations/20200819145320-vector-clock.js | 12 ++--- creator-node/src/config.js | 8 +-- creator-node/src/models/audiususer.js | 3 ++ creator-node/src/models/cNodeUser.js | 3 ++ creator-node/src/models/file.js | 3 ++ creator-node/src/models/track.js | 3 ++ 8 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 creator-node/scripts/db.js diff --git a/creator-node/scripts/clock-data-migration.js b/creator-node/scripts/clock-data-migration.js index aeef9d996e7..4bd91e22a11 100644 --- a/creator-node/scripts/clock-data-migration.js +++ b/creator-node/scripts/clock-data-migration.js @@ -1,7 +1,7 @@ const fs = require('fs-extra') const assert = require('assert') -const models = require('../src/models') +const initDB = require('./db.js') const discprovUsersFilePath = './discprov-users.txt' @@ -9,6 +9,12 @@ const discprovUsersFilePath = './discprov-users.txt' const usersToRSetMap = {} const nodeToUsersMap = {} +const endpointToDbUrl = { + 'http://cn1_creator-node_1:4000': 'postgres://postgres:postgres@127.0.0.1:4432/audius_creator_node', + 'http://cn2_creator-node_1:4001': 'postgres://postgres:postgres@127.0.0.1:4433/audius_creator_node', + 'http://cn3_creator-node_1:4002': 'postgres://postgres:postgres@127.0.0.1:4434/audius_creator_node' +} + const buildNodeToUsersMap = async () => { const fileData = fs.readFileSync(discprovUsersFilePath, 'utf8') @@ -49,13 +55,24 @@ const populateClockVals = async () => { const nodes = Object.keys(nodeToUsersMap) for (const node of nodes) { + // init DB instances + const dbUrl = endpointToDbUrl[node] + const models = await initDB(dbUrl) + for (const { userId, secondaries } of nodeToUsersMap[node]) { - // console.log(node, userId, secondaries) - // const { userId, secondaries } = userObj + console.log(node, dbUrl, userId, secondaries) + + const transaction = await models.sequelize.transaction() + + const audiusUser = await models.AudiusUser.findAll({ + where: { "blockchainId": userId }, + transaction + }) + + console.log(audiusUser) + console.log('\n\n\n\n\n') - // const transaction = await models.sequelize.transaction() - const resp = await models.CNodeUser.find() - console.log(resp) + await transaction.commit() } } } diff --git a/creator-node/scripts/db.js b/creator-node/scripts/db.js new file mode 100644 index 00000000000..701d2ebbd37 --- /dev/null +++ b/creator-node/scripts/db.js @@ -0,0 +1,51 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const Sequelize = require('sequelize') + +const modelsDirName = path.resolve('../src/models') + +const basename = path.basename('index.js') + +const initDB = async (dbUrl) => { + const db = {} + + const sequelize = new Sequelize(dbUrl, { + logging: true, + operatorsAliases: false, + // dialectOptions: { + // idleTimeoutMillis: 500, + // connectionTimeoutMillis: 500 + // }, + pool: { + max: 100, + min: 5, + acquire: 60000, + idle: 10000 + } + }) + + fs + .readdirSync(modelsDirName) + .filter(file => { + return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js') + }) + .forEach(file => { + const model = sequelize['import'](path.join(modelsDirName, file)) + db[model.name] = model + }) + + Object.keys(db).forEach(modelName => { + if (db[modelName].associate) { + db[modelName].associate(db) + } + }) + + db.sequelize = sequelize + db.Sequelize = Sequelize + + return db +} + +module.exports = initDB diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200819145320-vector-clock.js index 98fb472cc19..09b8e22bf9d 100644 --- a/creator-node/sequelize/migrations/20200819145320-vector-clock.js +++ b/creator-node/sequelize/migrations/20200819145320-vector-clock.js @@ -5,23 +5,19 @@ module.exports = { // Add 'clock2' column to all 4 data tables - TESTING ONLY await queryInterface.addColumn('CNodeUsers', 'clock2', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false }) await queryInterface.addColumn('AudiusUsers', 'clock2', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false }) await queryInterface.addColumn('Tracks', 'clock2', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false }) await queryInterface.addColumn('Files', 'clock2', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false }) diff --git a/creator-node/src/config.js b/creator-node/src/config.js index 5d4382d2733..c403add28aa 100644 --- a/creator-node/src/config.js +++ b/creator-node/src/config.js @@ -386,10 +386,10 @@ const config = convict({ */ // TODO(DM) - remove these defaults -const defaultConfigExists = fs.existsSync('../default-config.json') -if (defaultConfigExists) config.loadFile('../default-config.json') +const defaultConfigExists = fs.existsSync('default-config.json') +if (defaultConfigExists) config.loadFile('default-config.json') -if (fs.existsSync('../eth-contract-config.json')) { +if (fs.existsSync('eth-contract-config.json')) { let ethContractConfig = require('../eth-contract-config.json') config.load({ 'ethTokenAddress': ethContractConfig.audiusTokenAddress, @@ -399,7 +399,7 @@ if (fs.existsSync('../eth-contract-config.json')) { }) } -if (fs.existsSync('../contract-config.json')) { +if (fs.existsSync('contract-config.json')) { const dataContractConfig = require('../contract-config.json') config.load({ 'dataRegistryAddress': dataContractConfig.registryAddress diff --git a/creator-node/src/models/audiususer.js b/creator-node/src/models/audiususer.js index d5e9c8416c0..409d255729e 100644 --- a/creator-node/src/models/audiususer.js +++ b/creator-node/src/models/audiususer.js @@ -35,6 +35,9 @@ module.exports = (sequelize, DataTypes) => { clock: { type: DataTypes.INTEGER, allowNull: false + }, + clock2: { + type: DataTypes.INTEGER } }, {}) AudiusUser.associate = function (models) { diff --git a/creator-node/src/models/cNodeUser.js b/creator-node/src/models/cNodeUser.js index bcdf0ca1bb1..fb2151422fd 100644 --- a/creator-node/src/models/cNodeUser.js +++ b/creator-node/src/models/cNodeUser.js @@ -25,6 +25,9 @@ module.exports = (sequelize, DataTypes) => { clock: { type: DataTypes.INTEGER, allowNull: false + }, + clock2: { + type: DataTypes.INTEGER } }, {}) diff --git a/creator-node/src/models/file.js b/creator-node/src/models/file.js index 27dfdb4da0a..50300e2f45b 100644 --- a/creator-node/src/models/file.js +++ b/creator-node/src/models/file.js @@ -53,6 +53,9 @@ module.exports = (sequelize, DataTypes) => { clock: { type: DataTypes.INTEGER, allowNull: false + }, + clock2: { + type: DataTypes.INTEGER } }, { indexes: [ diff --git a/creator-node/src/models/track.js b/creator-node/src/models/track.js index fdab2578d73..c9f2278f496 100644 --- a/creator-node/src/models/track.js +++ b/creator-node/src/models/track.js @@ -32,6 +32,9 @@ module.exports = (sequelize, DataTypes) => { clock: { type: DataTypes.INTEGER, allowNull: false + }, + clock2: { + type: DataTypes.INTEGER } }, {}) From 249d484aec1e2242b5bc8de2c52e40bfdc5a7d58 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Mon, 7 Sep 2020 22:40:34 +0000 Subject: [PATCH 09/53] WIP --- creator-node/scripts/clock-data-migration.js | 31 ++++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/creator-node/scripts/clock-data-migration.js b/creator-node/scripts/clock-data-migration.js index 4bd91e22a11..80d7fbdc259 100644 --- a/creator-node/scripts/clock-data-migration.js +++ b/creator-node/scripts/clock-data-migration.js @@ -62,17 +62,36 @@ const populateClockVals = async () => { for (const { userId, secondaries } of nodeToUsersMap[node]) { console.log(node, dbUrl, userId, secondaries) - const transaction = await models.sequelize.transaction() + const transaction = null // await models.sequelize.transaction() - const audiusUser = await models.AudiusUser.findAll({ - where: { "blockchainId": userId }, + const audiusUsers = await models.AudiusUser.findAll({ + where: { blockchainId: userId }, transaction }) + console.log(audiusUsers) - console.log(audiusUser) - console.log('\n\n\n\n\n') + if (!audiusUsers) continue - await transaction.commit() + const cnodeUserUUID = audiusUsers[0].cnodeUserUUID + + const cnodeUsers = await models.CNodeUser.findAll({ + where: { cnodeUserUUID }, + transaction + }) + const tracks = await models.Track.findAll({ + where: { cnodeUserUUID }, + transaction + }) + const files = await models.File.findAll({ + where: { cnodeUserUUID }, + transaction + }) + + // order all rows from Files, Tracks, AudiusUsers by "updatedAt" asc + + // if dupe timestamps -> log out (this should never happen) + + // await transaction.commit() } } } From 60da211497b8c5c1e09ff4bfedc408f7e807ed33 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Tue, 8 Sep 2020 15:35:17 +0000 Subject: [PATCH 10/53] Working data migration --- creator-node/scripts/clock-data-migration.js | 154 ++++++++++++++----- 1 file changed, 118 insertions(+), 36 deletions(-) diff --git a/creator-node/scripts/clock-data-migration.js b/creator-node/scripts/clock-data-migration.js index 80d7fbdc259..d1490da0add 100644 --- a/creator-node/scripts/clock-data-migration.js +++ b/creator-node/scripts/clock-data-migration.js @@ -1,9 +1,14 @@ const fs = require('fs-extra') const assert = require('assert') +const axios = require('axios') const initDB = require('./db.js') - +/** + * select user_id, wallet, creator_node_endpoint from users + * where is_current = true and is_creator = true + * and creator_node_endpoint is not null; + */ const discprovUsersFilePath = './discprov-users.txt' const usersToRSetMap = {} @@ -22,11 +27,10 @@ const buildNodeToUsersMap = async () => { for (const fileDataLine of fileDataLines) { const [userId, wallet, endpointStr] = fileDataLine.split('\t') - // console.log(userId, endpointStr) - - usersToRSetMap[userId] = { wallet, endpointStr } + usersToRSetMap[userId] = { wallet, endpointStr } const [primary, ...secondaries] = endpointStr.split(',') + if (nodeToUsersMap[primary]) { nodeToUsersMap[primary].push({ userId, secondaries }) } else { @@ -60,41 +64,119 @@ const populateClockVals = async () => { const models = await initDB(dbUrl) for (const { userId, secondaries } of nodeToUsersMap[node]) { - console.log(node, dbUrl, userId, secondaries) - - const transaction = null // await models.sequelize.transaction() - - const audiusUsers = await models.AudiusUser.findAll({ - where: { blockchainId: userId }, - transaction - }) - console.log(audiusUsers) - - if (!audiusUsers) continue - - const cnodeUserUUID = audiusUsers[0].cnodeUserUUID - - const cnodeUsers = await models.CNodeUser.findAll({ - where: { cnodeUserUUID }, - transaction - }) - const tracks = await models.Track.findAll({ - where: { cnodeUserUUID }, - transaction - }) - const files = await models.File.findAll({ - where: { cnodeUserUUID }, - transaction - }) - - // order all rows from Files, Tracks, AudiusUsers by "updatedAt" asc - - // if dupe timestamps -> log out (this should never happen) - - // await transaction.commit() + const transaction = await models.sequelize.transaction() + + try { + // console.log('\n\n\n') + const audiusUsers = await models.AudiusUser.findAll({ + where: { blockchainId: userId }, + transaction + }) + + let cnodeUserUUID + if (audiusUsers && audiusUsers.length > 0) { + cnodeUserUUID = audiusUsers[0].cnodeUserUUID + + const tracks = await models.Track.findAll({ + where: { cnodeUserUUID }, + transaction + }) + const files = await models.File.findAll({ + where: { cnodeUserUUID }, + transaction + }) + + // Aggregate all data in array + const data = [] + for (const audiusUser of audiusUsers) data.push([audiusUser, 'audiusUser']) + for (const track of tracks) data.push([track, 'track']) + for (const file of files) data.push([file, `file - ${file.dataValues.type}`]) + // for (const datum of data) { + // const entry = datum[0].dataValues + // // console.log(`SIDTEST DATA ENTRY: ${entry.createdAt} - ${entry.clock}`) + // } + + // order all rows from Files, Tracks, AudiusUsers by "created" ASC (compares times in milliseconds) + data.sort((a, b) => { + const dA = new Date(a[0].dataValues.createdAt).getTime() + const dB = new Date(b[0].dataValues.createdAt).getTime() + const diff = dA - dB + return diff + }) + // for (const datum of data) { + // const entry = datum[0].dataValues + // // console.log(`SIDTESTSORTED DATA ENTRY: ${entry.createdAt} - ${entry.clock} - ${datum[1]}`) + // } + + // Update each data table row with new clock2 value + let clockCounter = 0 + for (const datum of data) { + const datumUpdateResp = await datum[0].update( + { clock2: ++clockCounter }, + { transaction } + ) + } + + // Update cnodeUser row with final clock2 value + const numRowsChanged = await models.CNodeUser.update( + { clock2: clockCounter }, + { + where: { cnodeUserUUID }, + transaction + } + ) + if (!numRowsChanged) { + throw new Error('CNodeUser update failed') + } + } + + await transaction.commit() + + // force sync secondaries + if (cnodeUserUUID) { + const cnodeUserWallet = usersToRSetMap[userId].wallet + await triggerSecondarySyncs(node, secondaries, cnodeUserWallet) + await timeout(2000) + } + } catch (e) { + console.error(`SIDTESTERROR`, e) + await transaction.rollback() + } } } } buildNodeToUsersMap() populateClockVals() + + +/** + * Tell all secondaries to sync against self. + * @dev - Is not a middleware so it can be run before responding to client. + */ +async function triggerSecondarySyncs (primary, secondaries, wallet) { + // TODO - throw if resp fails + // TODO - modify sync to not fail on equal blocknumber + const resp = await Promise.all(secondaries.map( + async (secondary) => { + if (!secondary) return + + const axiosReq = { + baseURL: secondary, + url: '/sync', + method: 'post', + data: { + wallet: [wallet], + creator_node_endpoint: primary, + immediate: true + } + } + return axios(axiosReq) + } + )) +} + +async function timeout (ms) { + console.log(`starting timeout of ${ms}`) + return new Promise(resolve => setTimeout(resolve, ms)) +} \ No newline at end of file From f8a23d29ff36e3c2fa28c8bf16e9b97dd41b8a51 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 11 Sep 2020 14:25:37 +0000 Subject: [PATCH 11/53] WIP data migration --- creator-node/scripts/clock-data-migration.js | 78 ++++++++++++------- .../migrations/20200819145320-vector-clock.js | 50 ++++++------ 2 files changed, 75 insertions(+), 53 deletions(-) diff --git a/creator-node/scripts/clock-data-migration.js b/creator-node/scripts/clock-data-migration.js index d1490da0add..bdc8740ca0c 100644 --- a/creator-node/scripts/clock-data-migration.js +++ b/creator-node/scripts/clock-data-migration.js @@ -9,7 +9,7 @@ const initDB = require('./db.js') * where is_current = true and is_creator = true * and creator_node_endpoint is not null; */ -const discprovUsersFilePath = './discprov-users.txt' +const discprovUsersFilePath = './discprov-users-PROD.txt' const usersToRSetMap = {} const nodeToUsersMap = {} @@ -19,17 +19,24 @@ const endpointToDbUrl = { 'http://cn2_creator-node_1:4001': 'postgres://postgres:postgres@127.0.0.1:4433/audius_creator_node', 'http://cn3_creator-node_1:4002': 'postgres://postgres:postgres@127.0.0.1:4434/audius_creator_node' } +const endpointToDbUrlProd = { + 'https://creatornode.audius.co': 'postgres://creator_1:postgres@127.0.0.1:5475/audius_creator_node', + 'https://creatornode2.audius.co': '', + 'https://creatornode3.audius.co': '' +} const buildNodeToUsersMap = async () => { const fileData = fs.readFileSync(discprovUsersFilePath, 'utf8') - const fileDataLines = fileData.split('\n') + let count = 0 for (const fileDataLine of fileDataLines) { const [userId, wallet, endpointStr] = fileDataLine.split('\t') + // console.log(`SIDTEST ROW #${++count}`, userId, wallet, endpointStr) usersToRSetMap[userId] = { wallet, endpointStr } const [primary, ...secondaries] = endpointStr.split(',') + // console.log(` ENDPOINT SPLIT`, primary, secondaries) if (nodeToUsersMap[primary]) { nodeToUsersMap[primary].push({ userId, secondaries }) @@ -38,8 +45,11 @@ const buildNodeToUsersMap = async () => { } } - // console.log('\n\n') - // console.log(JSON.stringify(nodeToUsersMap, undefined, 2)) + count = 0 + // for (const user of nodeToUsersMap['https://creatornode.audius.co']) { + // if (++count > 300) break + // console.log(`SIDTEST CN1 || USERID: `, user.userId, " || SECONDARIES: ", JSON.stringify(user.secondaries)) + // } assert.equal(fileDataLines.length, (Object.keys(usersToRSetMap)).length) } @@ -58,16 +68,21 @@ for (primary of primaries) const populateClockVals = async () => { const nodes = Object.keys(nodeToUsersMap) + // TODO - modify all queries to add SELECT FOR UPDATE and hold lock until commit for (const node of nodes) { + if (node != 'https://creatornode.audius.co') continue // init DB instances - const dbUrl = endpointToDbUrl[node] + const dbUrl = endpointToDbUrlProd[node] const models = await initDB(dbUrl) + console.log(`number of users on ${node}: ${nodeToUsersMap[node].length}`) for (const { userId, secondaries } of nodeToUsersMap[node]) { + console.log('\n\n\n\n') + const start = Date.now() + const transaction = await models.sequelize.transaction() try { - // console.log('\n\n\n') const audiusUsers = await models.AudiusUser.findAll({ where: { blockchainId: userId }, transaction @@ -76,6 +91,14 @@ const populateClockVals = async () => { let cnodeUserUUID if (audiusUsers && audiusUsers.length > 0) { cnodeUserUUID = audiusUsers[0].cnodeUserUUID + + console.log(`userId: ${userId} || audiusUserUUID: ${audiusUsers[0].audiusUserUUID} || cnodeUserUUID: ${cnodeUserUUID}`) + + // Short circuit if audiusUser already has clock value + if (audiusUsers[0].clock != null) { + console.log(`audiusUser already has clock value of ${audiusUsers[0].clock}. Short-circuiting migration`) + continue + } const tracks = await models.Track.findAll({ where: { cnodeUserUUID }, @@ -88,13 +111,9 @@ const populateClockVals = async () => { // Aggregate all data in array const data = [] - for (const audiusUser of audiusUsers) data.push([audiusUser, 'audiusUser']) - for (const track of tracks) data.push([track, 'track']) - for (const file of files) data.push([file, `file - ${file.dataValues.type}`]) - // for (const datum of data) { - // const entry = datum[0].dataValues - // // console.log(`SIDTEST DATA ENTRY: ${entry.createdAt} - ${entry.clock}`) - // } + for (const audiusUser of audiusUsers) data.push([audiusUser, `audiusUserUUID ${audiusUser.dataValues.audiusUserUUID}`]) + for (const track of tracks) data.push([track, `trackUUID ${track.dataValues.trackUUID}`]) + for (const file of files) data.push([file, `fileUUId ${file.dataValues.fileUUID} - ${file.dataValues.type}`]) // order all rows from Files, Tracks, AudiusUsers by "created" ASC (compares times in milliseconds) data.sort((a, b) => { @@ -103,23 +122,19 @@ const populateClockVals = async () => { const diff = dA - dB return diff }) - // for (const datum of data) { - // const entry = datum[0].dataValues - // // console.log(`SIDTESTSORTED DATA ENTRY: ${entry.createdAt} - ${entry.clock} - ${datum[1]}`) - // } - // Update each data table row with new clock2 value + // Update each data table row with new clock value let clockCounter = 0 for (const datum of data) { - const datumUpdateResp = await datum[0].update( - { clock2: ++clockCounter }, + await datum[0].update( + { clock: ++clockCounter }, { transaction } ) } - - // Update cnodeUser row with final clock2 value + + // Update cnodeUser row with final clock value const numRowsChanged = await models.CNodeUser.update( - { clock2: clockCounter }, + { clock: clockCounter }, { where: { cnodeUserUUID }, transaction @@ -128,28 +143,33 @@ const populateClockVals = async () => { if (!numRowsChanged) { throw new Error('CNodeUser update failed') } + } else { + console.log(`\n\n\nuserId: ${userId} || no audiusUser found`) } await transaction.commit() // force sync secondaries - if (cnodeUserUUID) { - const cnodeUserWallet = usersToRSetMap[userId].wallet - await triggerSecondarySyncs(node, secondaries, cnodeUserWallet) - await timeout(2000) - } + // if (cnodeUserUUID) { + // const cnodeUserWallet = usersToRSetMap[userId].wallet + // await triggerSecondarySyncs(node, secondaries, cnodeUserWallet) + // await timeout(2000) + // } } catch (e) { console.error(`SIDTESTERROR`, e) await transaction.rollback() } + + const durationMs = Date.now() - start + console.log(`USER ROUTE TIME (sec): ${Math.floor(durationMs / 1000)}`) } } + console.log('populateClockVals() COMPLETE') } buildNodeToUsersMap() populateClockVals() - /** * Tell all secondaries to sync against self. * @dev - Is not a middleware so it can be run before responding to client. diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200819145320-vector-clock.js index 09b8e22bf9d..62c3a95ccb0 100644 --- a/creator-node/sequelize/migrations/20200819145320-vector-clock.js +++ b/creator-node/sequelize/migrations/20200819145320-vector-clock.js @@ -2,45 +2,47 @@ module.exports = { up: async (queryInterface, Sequelize) => { + // const transaction = await queryInterface.sequelize.transaction() + // Add 'clock2' column to all 4 data tables - TESTING ONLY - await queryInterface.addColumn('CNodeUsers', 'clock2', { - type: Sequelize.INTEGER, - unique: false - }) - await queryInterface.addColumn('AudiusUsers', 'clock2', { - type: Sequelize.INTEGER, - unique: false - }) - await queryInterface.addColumn('Tracks', 'clock2', { - type: Sequelize.INTEGER, - unique: false - }) - await queryInterface.addColumn('Files', 'clock2', { - type: Sequelize.INTEGER, - unique: false - }) + // await queryInterface.addColumn('CNodeUsers', 'clock2', { + // type: Sequelize.INTEGER, + // unique: false + // }) + // await queryInterface.addColumn('AudiusUsers', 'clock2', { + // type: Sequelize.INTEGER, + // unique: false + // }) + // await queryInterface.addColumn('Tracks', 'clock2', { + // type: Sequelize.INTEGER, + // unique: false + // }) + // await queryInterface.addColumn('Files', 'clock2', { + // type: Sequelize.INTEGER, + // unique: false + // }) // Add 'clock' column to all 4 data tables await queryInterface.addColumn('CNodeUsers', 'clock', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false + // allowNull: false }) await queryInterface.addColumn('AudiusUsers', 'clock', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false + // allowNull: false }) await queryInterface.addColumn('Tracks', 'clock', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false + // allowNull: false }) await queryInterface.addColumn('Files', 'clock', { type: Sequelize.INTEGER, - unique: false, - allowNull: false + unique: false + // allowNull: false }) // Add composite uniqueness constraint on (cnodeUserUUID, clock) in each table From d4d107aa81e543f37e6f9352352505b025f1fbee Mon Sep 17 00:00:00 2001 From: SidSethi Date: Tue, 15 Sep 2020 01:21:03 +0000 Subject: [PATCH 12/53] migration WIP --- .../migrations/20200819145320-vector-clock.js | 161 ++++++++++-------- .../20200911203547-add-clocks-table.js | 23 +++ 2 files changed, 113 insertions(+), 71 deletions(-) create mode 100644 creator-node/sequelize/migrations/20200911203547-add-clocks-table.js diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200819145320-vector-clock.js index 62c3a95ccb0..07c511c09b3 100644 --- a/creator-node/sequelize/migrations/20200819145320-vector-clock.js +++ b/creator-node/sequelize/migrations/20200819145320-vector-clock.js @@ -1,85 +1,27 @@ 'use strict' +/** + * Content Tables = AudiusUsers, Tracks, Files + * CNodeUsers Table considered a Reference Table only + */ + module.exports = { up: async (queryInterface, Sequelize) => { - // const transaction = await queryInterface.sequelize.transaction() + const transaction = await queryInterface.sequelize.transaction() // Add 'clock2' column to all 4 data tables - TESTING ONLY - // await queryInterface.addColumn('CNodeUsers', 'clock2', { - // type: Sequelize.INTEGER, - // unique: false - // }) - // await queryInterface.addColumn('AudiusUsers', 'clock2', { - // type: Sequelize.INTEGER, - // unique: false - // }) - // await queryInterface.addColumn('Tracks', 'clock2', { - // type: Sequelize.INTEGER, - // unique: false - // }) - // await queryInterface.addColumn('Files', 'clock2', { - // type: Sequelize.INTEGER, - // unique: false - // }) - + await addClock2Column(queryInterface, Sequelize, transaction, true) // Add 'clock' column to all 4 data tables - await queryInterface.addColumn('CNodeUsers', 'clock', { - type: Sequelize.INTEGER, - unique: false - // allowNull: false - }) - await queryInterface.addColumn('AudiusUsers', 'clock', { - type: Sequelize.INTEGER, - unique: false - // allowNull: false - }) - await queryInterface.addColumn('Tracks', 'clock', { - type: Sequelize.INTEGER, - unique: false - // allowNull: false - }) - await queryInterface.addColumn('Files', 'clock', { - type: Sequelize.INTEGER, - unique: false - // allowNull: false - }) + await addClockColumn(queryInterface, Sequelize, transaction, false) - // Add composite uniqueness constraint on (cnodeUserUUID, clock) in each table - await queryInterface.addConstraint( - 'CNodeUsers', - { - type: 'UNIQUE', - fields: ['cnodeUserUUID', 'clock'], - name: 'CNodeUsers_unique_constraint_(cnodeUserUUID,clock)' - } - ) - await queryInterface.addConstraint( - 'AudiusUsers', - { - type: 'UNIQUE', - fields: ['cnodeUserUUID', 'clock'], - name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,clock)' - } - ) - await queryInterface.addConstraint( - 'Tracks', - { - type: 'UNIQUE', - fields: ['cnodeUserUUID', 'clock'], - name: 'Tracks_unique_constraint_(cnodeUserUUID,clock)' - } - ) - // await queryInterface.addConstraint( - // 'Files', - // { - // type: 'UNIQUE', - // fields: ['cnodeUserUUID', 'clock'], - // name: 'Files_unique_constraint_(cnodeUserUUID,clock)' - // } - // ) + // Add composite uniqueness constraint on (cnodeUserUUID, clock) to all Content Tables + await addConstraints(queryInterface, Sequelize, transaction) + + await transaction.commit() }, + // TODO down: async (queryInterface, Sequelize) => { // Remove uniqueness constraints on (cnodeUserUUID, clock) on all 4 tables await queryInterface.removeConstraint( @@ -102,3 +44,80 @@ module.exports = { await queryInterface.removeColumn('Files', 'clock') } } + +async function addClockColumn (queryInterface, Sequelize, transaction, allowNull) { + await queryInterface.addColumn('CNodeUsers', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) + await queryInterface.addColumn('AudiusUsers', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) + await queryInterface.addColumn('Tracks', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) + await queryInterface.addColumn('Files', 'clock', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) +} + +async function addClock2Column (queryInterface, Sequelize, transaction, allowNull) { + await queryInterface.addColumn('CNodeUsers', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) + await queryInterface.addColumn('AudiusUsers', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) + await queryInterface.addColumn('Tracks', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) + await queryInterface.addColumn('Files', 'clock2', { + type: Sequelize.INTEGER, + unique: false, + allowNull + }, { transaction }) +} + +// Add uniqueness constraint on composite (cnodeUserUUId, clock) to Content Tables +async function addConstraints (queryInterface, Sequelize, transaction) { + await queryInterface.addConstraint( + 'AudiusUsers', + { + type: 'UNIQUE', + fields: ['cnodeUserUUID', 'clock'], + name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,clock)', + transaction + } + ) + await queryInterface.addConstraint( + 'Tracks', + { + type: 'UNIQUE', + fields: ['cnodeUserUUID', 'clock'], + name: 'Tracks_unique_constraint_(cnodeUserUUID,clock)', + transaction + } + ) + await queryInterface.addConstraint( + 'Files', + { + type: 'UNIQUE', + fields: ['cnodeUserUUID', 'clock'], + name: 'Files_unique_constraint_(cnodeUserUUID,clock)', + transaction + } + ) +} \ No newline at end of file diff --git a/creator-node/sequelize/migrations/20200911203547-add-clocks-table.js b/creator-node/sequelize/migrations/20200911203547-add-clocks-table.js new file mode 100644 index 00000000000..9ef68be8da2 --- /dev/null +++ b/creator-node/sequelize/migrations/20200911203547-add-clocks-table.js @@ -0,0 +1,23 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + /* + Add altering commands here. + Return a promise to correctly handle asynchronicity. + + Example: + return queryInterface.createTable('users', { id: Sequelize.INTEGER }); + */ + }, + + down: (queryInterface, Sequelize) => { + /* + Add reverting commands here. + Return a promise to correctly handle asynchronicity. + + Example: + return queryInterface.dropTable('users'); + */ + } +}; From cc1612a734ed77be7ee6de18a5fb488bae1473ad Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 17 Sep 2020 19:13:13 +0000 Subject: [PATCH 13/53] Working new clockwork implementation + all tests passing --- creator-node/scripts/clock-data-migration.js | 404 +++++++++--------- creator-node/scripts/db.js | 102 ++--- creator-node/scripts/run-tests.sh | 2 +- .../migrations/20200819145320-vector-clock.js | 51 ++- .../20200911203547-add-clocks-table.js | 23 - creator-node/src/clockManager.js | 43 ++ ...UserClock.test.js => clockManager.test.js} | 26 +- creator-node/src/fileManager.js | 67 +-- creator-node/src/models/clockRecord.js | 49 +++ creator-node/src/models/file.js | 1 + creator-node/src/routes/audiusUsers.js | 62 ++- creator-node/src/routes/files.js | 84 ++-- creator-node/src/routes/tracks.js | 287 +++++++------ creator-node/src/routes/users.js | 3 +- creator-node/src/segmentDuration.js | 4 +- .../utils/incrementAndFetchCNodeUserClock.js | 32 -- .../{audiusUsers.js => audiusUsers.test.js} | 15 +- .../{expressApp.js => expressApp.test.js} | 2 +- .../test/{ffmpeg.js => ffmpeg.test.js} | 3 - .../{fileManager.js => fileManager.test.js} | 58 +-- .../test/{hashids.js => hashids.test.js} | 0 .../{resizeImage.js => resizeImage.test.js} | 0 creator-node/test/tracks.js | 136 +++--- creator-node/test/users.js | 4 +- 24 files changed, 742 insertions(+), 716 deletions(-) delete mode 100644 creator-node/sequelize/migrations/20200911203547-add-clocks-table.js create mode 100644 creator-node/src/clockManager.js rename creator-node/src/{utils/incrementAndFetchCNodeUserClock.test.js => clockManager.test.js} (78%) create mode 100644 creator-node/src/models/clockRecord.js delete mode 100644 creator-node/src/utils/incrementAndFetchCNodeUserClock.js rename creator-node/test/{audiusUsers.js => audiusUsers.test.js} (93%) rename creator-node/test/{expressApp.js => expressApp.test.js} (96%) rename creator-node/test/{ffmpeg.js => ffmpeg.test.js} (97%) rename creator-node/test/{fileManager.js => fileManager.test.js} (81%) rename creator-node/test/{hashids.js => hashids.test.js} (100%) rename creator-node/test/{resizeImage.js => resizeImage.test.js} (100%) diff --git a/creator-node/scripts/clock-data-migration.js b/creator-node/scripts/clock-data-migration.js index bdc8740ca0c..a2148906eb4 100644 --- a/creator-node/scripts/clock-data-migration.js +++ b/creator-node/scripts/clock-data-migration.js @@ -1,202 +1,202 @@ -const fs = require('fs-extra') -const assert = require('assert') -const axios = require('axios') - -const initDB = require('./db.js') - -/** - * select user_id, wallet, creator_node_endpoint from users - * where is_current = true and is_creator = true - * and creator_node_endpoint is not null; - */ -const discprovUsersFilePath = './discprov-users-PROD.txt' - -const usersToRSetMap = {} -const nodeToUsersMap = {} - -const endpointToDbUrl = { - 'http://cn1_creator-node_1:4000': 'postgres://postgres:postgres@127.0.0.1:4432/audius_creator_node', - 'http://cn2_creator-node_1:4001': 'postgres://postgres:postgres@127.0.0.1:4433/audius_creator_node', - 'http://cn3_creator-node_1:4002': 'postgres://postgres:postgres@127.0.0.1:4434/audius_creator_node' -} -const endpointToDbUrlProd = { - 'https://creatornode.audius.co': 'postgres://creator_1:postgres@127.0.0.1:5475/audius_creator_node', - 'https://creatornode2.audius.co': '', - 'https://creatornode3.audius.co': '' -} - -const buildNodeToUsersMap = async () => { - const fileData = fs.readFileSync(discprovUsersFilePath, 'utf8') - const fileDataLines = fileData.split('\n') - - let count = 0 - for (const fileDataLine of fileDataLines) { - const [userId, wallet, endpointStr] = fileDataLine.split('\t') - // console.log(`SIDTEST ROW #${++count}`, userId, wallet, endpointStr) - - usersToRSetMap[userId] = { wallet, endpointStr } - const [primary, ...secondaries] = endpointStr.split(',') - // console.log(` ENDPOINT SPLIT`, primary, secondaries) - - if (nodeToUsersMap[primary]) { - nodeToUsersMap[primary].push({ userId, secondaries }) - } else { - nodeToUsersMap[primary] = [{ userId, secondaries }] - } - } - - count = 0 - // for (const user of nodeToUsersMap['https://creatornode.audius.co']) { - // if (++count > 300) break - // console.log(`SIDTEST CN1 || USERID: `, user.userId, " || SECONDARIES: ", JSON.stringify(user.secondaries)) - // } - - assert.equal(fileDataLines.length, (Object.keys(usersToRSetMap)).length) -} - -/** -for (primary of primaries) - for (user_id of user_ids) - update clock state on primary (in transaction) - select cnodeUserUUID from AudiusUsers for blockchainId - select all rows in Files, Tracks, AudiusUsers for cnodeUserUUID - order all data in time order asc - assign auto-inc clcokval to each row from 1 - assign final clockval to cnodeUsers row - force sync secondaries against primary - */ -const populateClockVals = async () => { - const nodes = Object.keys(nodeToUsersMap) - - // TODO - modify all queries to add SELECT FOR UPDATE and hold lock until commit - for (const node of nodes) { - if (node != 'https://creatornode.audius.co') continue - // init DB instances - const dbUrl = endpointToDbUrlProd[node] - const models = await initDB(dbUrl) - - console.log(`number of users on ${node}: ${nodeToUsersMap[node].length}`) - for (const { userId, secondaries } of nodeToUsersMap[node]) { - console.log('\n\n\n\n') - const start = Date.now() - - const transaction = await models.sequelize.transaction() - - try { - const audiusUsers = await models.AudiusUser.findAll({ - where: { blockchainId: userId }, - transaction - }) - - let cnodeUserUUID - if (audiusUsers && audiusUsers.length > 0) { - cnodeUserUUID = audiusUsers[0].cnodeUserUUID - - console.log(`userId: ${userId} || audiusUserUUID: ${audiusUsers[0].audiusUserUUID} || cnodeUserUUID: ${cnodeUserUUID}`) - - // Short circuit if audiusUser already has clock value - if (audiusUsers[0].clock != null) { - console.log(`audiusUser already has clock value of ${audiusUsers[0].clock}. Short-circuiting migration`) - continue - } - - const tracks = await models.Track.findAll({ - where: { cnodeUserUUID }, - transaction - }) - const files = await models.File.findAll({ - where: { cnodeUserUUID }, - transaction - }) - - // Aggregate all data in array - const data = [] - for (const audiusUser of audiusUsers) data.push([audiusUser, `audiusUserUUID ${audiusUser.dataValues.audiusUserUUID}`]) - for (const track of tracks) data.push([track, `trackUUID ${track.dataValues.trackUUID}`]) - for (const file of files) data.push([file, `fileUUId ${file.dataValues.fileUUID} - ${file.dataValues.type}`]) - - // order all rows from Files, Tracks, AudiusUsers by "created" ASC (compares times in milliseconds) - data.sort((a, b) => { - const dA = new Date(a[0].dataValues.createdAt).getTime() - const dB = new Date(b[0].dataValues.createdAt).getTime() - const diff = dA - dB - return diff - }) - - // Update each data table row with new clock value - let clockCounter = 0 - for (const datum of data) { - await datum[0].update( - { clock: ++clockCounter }, - { transaction } - ) - } - - // Update cnodeUser row with final clock value - const numRowsChanged = await models.CNodeUser.update( - { clock: clockCounter }, - { - where: { cnodeUserUUID }, - transaction - } - ) - if (!numRowsChanged) { - throw new Error('CNodeUser update failed') - } - } else { - console.log(`\n\n\nuserId: ${userId} || no audiusUser found`) - } - - await transaction.commit() - - // force sync secondaries - // if (cnodeUserUUID) { - // const cnodeUserWallet = usersToRSetMap[userId].wallet - // await triggerSecondarySyncs(node, secondaries, cnodeUserWallet) - // await timeout(2000) - // } - } catch (e) { - console.error(`SIDTESTERROR`, e) - await transaction.rollback() - } - - const durationMs = Date.now() - start - console.log(`USER ROUTE TIME (sec): ${Math.floor(durationMs / 1000)}`) - } - } - console.log('populateClockVals() COMPLETE') -} - -buildNodeToUsersMap() -populateClockVals() - -/** - * Tell all secondaries to sync against self. - * @dev - Is not a middleware so it can be run before responding to client. - */ -async function triggerSecondarySyncs (primary, secondaries, wallet) { - // TODO - throw if resp fails - // TODO - modify sync to not fail on equal blocknumber - const resp = await Promise.all(secondaries.map( - async (secondary) => { - if (!secondary) return - - const axiosReq = { - baseURL: secondary, - url: '/sync', - method: 'post', - data: { - wallet: [wallet], - creator_node_endpoint: primary, - immediate: true - } - } - return axios(axiosReq) - } - )) -} - -async function timeout (ms) { - console.log(`starting timeout of ${ms}`) - return new Promise(resolve => setTimeout(resolve, ms)) -} \ No newline at end of file +// const fs = require('fs-extra') +// const assert = require('assert') +// const axios = require('axios') + +// const initDB = require('./db.js') + +// /** +// * select user_id, wallet, creator_node_endpoint from users +// * where is_current = true and is_creator = true +// * and creator_node_endpoint is not null; +// */ +// const discprovUsersFilePath = './discprov-users-PROD.txt' + +// const usersToRSetMap = {} +// const nodeToUsersMap = {} + +// const endpointToDbUrl = { +// 'http://cn1_creator-node_1:4000': 'postgres://postgres:postgres@127.0.0.1:4432/audius_creator_node', +// 'http://cn2_creator-node_1:4001': 'postgres://postgres:postgres@127.0.0.1:4433/audius_creator_node', +// 'http://cn3_creator-node_1:4002': 'postgres://postgres:postgres@127.0.0.1:4434/audius_creator_node' +// } +// const endpointToDbUrlProd = { +// 'https://creatornode.audius.co': 'postgres://creator_1:postgres@127.0.0.1:5475/audius_creator_node', +// 'https://creatornode2.audius.co': '', +// 'https://creatornode3.audius.co': '' +// } + +// const buildNodeToUsersMap = async () => { +// const fileData = fs.readFileSync(discprovUsersFilePath, 'utf8') +// const fileDataLines = fileData.split('\n') + +// let count = 0 +// for (const fileDataLine of fileDataLines) { +// const [userId, wallet, endpointStr] = fileDataLine.split('\t') +// // console.log(`SIDTEST ROW #${++count}`, userId, wallet, endpointStr) + +// usersToRSetMap[userId] = { wallet, endpointStr } +// const [primary, ...secondaries] = endpointStr.split(',') +// // console.log(` ENDPOINT SPLIT`, primary, secondaries) + +// if (nodeToUsersMap[primary]) { +// nodeToUsersMap[primary].push({ userId, secondaries }) +// } else { +// nodeToUsersMap[primary] = [{ userId, secondaries }] +// } +// } + +// count = 0 +// // for (const user of nodeToUsersMap['https://creatornode.audius.co']) { +// // if (++count > 300) break +// // console.log(`SIDTEST CN1 || USERID: `, user.userId, " || SECONDARIES: ", JSON.stringify(user.secondaries)) +// // } + +// assert.equal(fileDataLines.length, (Object.keys(usersToRSetMap)).length) +// } + +// /** +// for (primary of primaries) +// for (user_id of user_ids) +// update clock state on primary (in transaction) +// select cnodeUserUUID from AudiusUsers for blockchainId +// select all rows in Files, Tracks, AudiusUsers for cnodeUserUUID +// order all data in time order asc +// assign auto-inc clcokval to each row from 1 +// assign final clockval to cnodeUsers row +// force sync secondaries against primary +// */ +// const populateClockVals = async () => { +// const nodes = Object.keys(nodeToUsersMap) + +// // TODO - modify all queries to add SELECT FOR UPDATE and hold lock until commit +// for (const node of nodes) { +// if (node != 'https://creatornode.audius.co') continue +// // init DB instances +// const dbUrl = endpointToDbUrlProd[node] +// const models = await initDB(dbUrl) + +// console.log(`number of users on ${node}: ${nodeToUsersMap[node].length}`) +// for (const { userId, secondaries } of nodeToUsersMap[node]) { +// console.log('\n\n\n\n') +// const start = Date.now() + +// const transaction = await models.sequelize.transaction() + +// try { +// const audiusUsers = await models.AudiusUser.findAll({ +// where: { blockchainId: userId }, +// transaction +// }) + +// let cnodeUserUUID +// if (audiusUsers && audiusUsers.length > 0) { +// cnodeUserUUID = audiusUsers[0].cnodeUserUUID + +// console.log(`userId: ${userId} || audiusUserUUID: ${audiusUsers[0].audiusUserUUID} || cnodeUserUUID: ${cnodeUserUUID}`) + +// // Short circuit if audiusUser already has clock value +// if (audiusUsers[0].clock != null) { +// console.log(`audiusUser already has clock value of ${audiusUsers[0].clock}. Short-circuiting migration`) +// continue +// } + +// const tracks = await models.Track.findAll({ +// where: { cnodeUserUUID }, +// transaction +// }) +// const files = await models.File.findAll({ +// where: { cnodeUserUUID }, +// transaction +// }) + +// // Aggregate all data in array +// const data = [] +// for (const audiusUser of audiusUsers) data.push([audiusUser, `audiusUserUUID ${audiusUser.dataValues.audiusUserUUID}`]) +// for (const track of tracks) data.push([track, `trackUUID ${track.dataValues.trackUUID}`]) +// for (const file of files) data.push([file, `fileUUId ${file.dataValues.fileUUID} - ${file.dataValues.type}`]) + +// // order all rows from Files, Tracks, AudiusUsers by "created" ASC (compares times in milliseconds) +// data.sort((a, b) => { +// const dA = new Date(a[0].dataValues.createdAt).getTime() +// const dB = new Date(b[0].dataValues.createdAt).getTime() +// const diff = dA - dB +// return diff +// }) + +// // Update each data table row with new clock value +// let clockCounter = 0 +// for (const datum of data) { +// await datum[0].update( +// { clock: ++clockCounter }, +// { transaction } +// ) +// } + +// // Update cnodeUser row with final clock value +// const numRowsChanged = await models.CNodeUser.update( +// { clock: clockCounter }, +// { +// where: { cnodeUserUUID }, +// transaction +// } +// ) +// if (!numRowsChanged) { +// throw new Error('CNodeUser update failed') +// } +// } else { +// console.log(`\n\n\nuserId: ${userId} || no audiusUser found`) +// } + +// await transaction.commit() + +// // force sync secondaries +// // if (cnodeUserUUID) { +// // const cnodeUserWallet = usersToRSetMap[userId].wallet +// // await triggerSecondarySyncs(node, secondaries, cnodeUserWallet) +// // await timeout(2000) +// // } +// } catch (e) { +// console.error(`SIDTESTERROR`, e) +// await transaction.rollback() +// } + +// const durationMs = Date.now() - start +// console.log(`USER ROUTE TIME (sec): ${Math.floor(durationMs / 1000)}`) +// } +// } +// console.log('populateClockVals() COMPLETE') +// } + +// buildNodeToUsersMap() +// populateClockVals() + +// /** +// * Tell all secondaries to sync against self. +// * @dev - Is not a middleware so it can be run before responding to client. +// */ +// async function triggerSecondarySyncs (primary, secondaries, wallet) { +// // TODO - throw if resp fails +// // TODO - modify sync to not fail on equal blocknumber +// const resp = await Promise.all(secondaries.map( +// async (secondary) => { +// if (!secondary) return + +// const axiosReq = { +// baseURL: secondary, +// url: '/sync', +// method: 'post', +// data: { +// wallet: [wallet], +// creator_node_endpoint: primary, +// immediate: true +// } +// } +// return axios(axiosReq) +// } +// )) +// } + +// async function timeout (ms) { +// console.log(`starting timeout of ${ms}`) +// return new Promise(resolve => setTimeout(resolve, ms)) +// } diff --git a/creator-node/scripts/db.js b/creator-node/scripts/db.js index 701d2ebbd37..06bdeef9372 100644 --- a/creator-node/scripts/db.js +++ b/creator-node/scripts/db.js @@ -1,51 +1,51 @@ -'use strict' - -const fs = require('fs') -const path = require('path') -const Sequelize = require('sequelize') - -const modelsDirName = path.resolve('../src/models') - -const basename = path.basename('index.js') - -const initDB = async (dbUrl) => { - const db = {} - - const sequelize = new Sequelize(dbUrl, { - logging: true, - operatorsAliases: false, - // dialectOptions: { - // idleTimeoutMillis: 500, - // connectionTimeoutMillis: 500 - // }, - pool: { - max: 100, - min: 5, - acquire: 60000, - idle: 10000 - } - }) - - fs - .readdirSync(modelsDirName) - .filter(file => { - return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js') - }) - .forEach(file => { - const model = sequelize['import'](path.join(modelsDirName, file)) - db[model.name] = model - }) - - Object.keys(db).forEach(modelName => { - if (db[modelName].associate) { - db[modelName].associate(db) - } - }) - - db.sequelize = sequelize - db.Sequelize = Sequelize - - return db -} - -module.exports = initDB +// 'use strict' + +// const fs = require('fs') +// const path = require('path') +// const Sequelize = require('sequelize') + +// const modelsDirName = path.resolve('../src/models') + +// const basename = path.basename('index.js') + +// const initDB = async (dbUrl) => { +// const db = {} + +// const sequelize = new Sequelize(dbUrl, { +// logging: true, +// operatorsAliases: false, +// // dialectOptions: { +// // idleTimeoutMillis: 500, +// // connectionTimeoutMillis: 500 +// // }, +// pool: { +// max: 100, +// min: 5, +// acquire: 60000, +// idle: 10000 +// } +// }) + +// fs +// .readdirSync(modelsDirName) +// .filter(file => { +// return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js') +// }) +// .forEach(file => { +// const model = sequelize['import'](path.join(modelsDirName, file)) +// db[model.name] = model +// }) + +// Object.keys(db).forEach(modelName => { +// if (db[modelName].associate) { +// db[modelName].associate(db) +// } +// }) + +// db.sequelize = sequelize +// db.Sequelize = Sequelize + +// return db +// } + +// module.exports = initDB diff --git a/creator-node/scripts/run-tests.sh b/creator-node/scripts/run-tests.sh index f9816abe58a..f2b029d40cd 100755 --- a/creator-node/scripts/run-tests.sh +++ b/creator-node/scripts/run-tests.sh @@ -36,7 +36,7 @@ run_unit_tests () { run_integration_tests () { set +e echo Running integration tests... - ./node_modules/mocha/bin/mocha --timeout 30000 --exit + ./node_modules/mocha/bin/mocha test/*.test.js --timeout 30000 --exit set -e } diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200819145320-vector-clock.js index 07c511c09b3..5eb161c36d0 100644 --- a/creator-node/sequelize/migrations/20200819145320-vector-clock.js +++ b/creator-node/sequelize/migrations/20200819145320-vector-clock.js @@ -9,15 +9,18 @@ module.exports = { up: async (queryInterface, Sequelize) => { const transaction = await queryInterface.sequelize.transaction() - // Add 'clock2' column to all 4 data tables - TESTING ONLY + // TODO remove - Add 'clock2' column to all 4 data tables await addClock2Column(queryInterface, Sequelize, transaction, true) - // Add 'clock' column to all 4 data tables + // Add 'clock' column to all 4 tables await addClockColumn(queryInterface, Sequelize, transaction, false) // Add composite uniqueness constraint on (cnodeUserUUID, clock) to all Content Tables - await addConstraints(queryInterface, Sequelize, transaction) - + await addUniquenessConstraints(queryInterface, Sequelize, transaction) + + // Create Clock table + await createClockRecordsTable(queryInterface, Sequelize, transaction) + await transaction.commit() }, @@ -92,7 +95,7 @@ async function addClock2Column (queryInterface, Sequelize, transaction, allowNul } // Add uniqueness constraint on composite (cnodeUserUUId, clock) to Content Tables -async function addConstraints (queryInterface, Sequelize, transaction) { +async function addUniquenessConstraints (queryInterface, Sequelize, transaction) { await queryInterface.addConstraint( 'AudiusUsers', { @@ -120,4 +123,40 @@ async function addConstraints (queryInterface, Sequelize, transaction) { transaction } ) -} \ No newline at end of file +} + +async function createClockRecordsTable (queryInterface, Sequelize, transaction) { + await queryInterface.createTable('ClockRecords', { + cnodeUserUUID: { + type: Sequelize.UUID, + primaryKey: true, // composite PK with clock + unique: false, + allowNull: false, + references: { + model: 'CNodeUsers', + key: 'cnodeUserUUID', + as: 'cnodeUserUUID' + }, + onDelete: 'RESTRICT' + }, + clock: { + type: Sequelize.INTEGER, + primaryKey: true, // composite PK with cnodeUserUUID + unique: false, + allowNull: false + }, + sourceTable: { + // TODO - if this doesn't work, use models/file.js:L46 + type: Sequelize.ENUM('AudiusUser', 'Track', 'File'), + allowNull: false + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE + } + }, { transaction }) +} diff --git a/creator-node/sequelize/migrations/20200911203547-add-clocks-table.js b/creator-node/sequelize/migrations/20200911203547-add-clocks-table.js deleted file mode 100644 index 9ef68be8da2..00000000000 --- a/creator-node/sequelize/migrations/20200911203547-add-clocks-table.js +++ /dev/null @@ -1,23 +0,0 @@ -'use strict'; - -module.exports = { - up: (queryInterface, Sequelize) => { - /* - Add altering commands here. - Return a promise to correctly handle asynchronicity. - - Example: - return queryInterface.createTable('users', { id: Sequelize.INTEGER }); - */ - }, - - down: (queryInterface, Sequelize) => { - /* - Add reverting commands here. - Return a promise to correctly handle asynchronicity. - - Example: - return queryInterface.dropTable('users'); - */ - } -}; diff --git a/creator-node/src/clockManager.js b/creator-node/src/clockManager.js new file mode 100644 index 00000000000..34df0c92a00 --- /dev/null +++ b/creator-node/src/clockManager.js @@ -0,0 +1,43 @@ +const models = require('./models') +const sequelize = models.sequelize + +// TODO consider adding hook to ensure write op can never set clockVal to anything <= current + +// TODO - any further constraint enforcement needed? +// - DataTables all FK to Clocks table +// - Clocks table has unique constraint +// - dataTables have unique C on (userId + clock) + +const updateClockInCNodeUserAndClockRecords = async (req, sourceTable, transaction) => { + const cnodeUserUUID = req.session.cnodeUserUUID + + // Increment CNodeUser.clock value by 1 + await models.CNodeUser.increment( + 'clock', /* fields */ + { + where: { cnodeUserUUID: req.session.cnodeUserUUID }, + by: 1, + transaction + } + ) + + // Add row in ClockRecords table using clock value from CNodeUser + await models.ClockRecord.create({ + cnodeUserUUID: req.session.cnodeUserUUID, + clock: sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`), + sourceTable + }, { transaction }) +} + +// source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause +const selectCNodeUserClockSubquery = (cnodeUserUUID) => { + return sequelize.dialect.QueryGenerator.selectQuery('CNodeUsers', { + attributes: ['clock'], + where: { cnodeUserUUID } + }).slice(0, -1) // removes trailing ';' +} + +module.exports = { + updateClockInCNodeUserAndClockRecords, + selectCNodeUserClockSubquery +} diff --git a/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js b/creator-node/src/clockManager.test.js similarity index 78% rename from creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js rename to creator-node/src/clockManager.test.js index 66b09e7af49..f9c95625a92 100644 --- a/creator-node/src/utils/incrementAndFetchCNodeUserClock.test.js +++ b/creator-node/src/clockManager.test.js @@ -2,12 +2,12 @@ const assert = require('assert') const proxyquire = require('proxyquire') const _ = require('lodash') -const models = require('../models') -const { createStarterCNodeUser } = require('../../test/lib/dataSeeds') -const { incrementAndFetchCNodeUserClock } = require('./incrementAndFetchCNodeUserClock') -const utils = require('../utils') +const models = require('./models') +const { createStarterCNodeUser } = require('../test/lib/dataSeeds') +const { incrementAndFetchCNodeUserClock, fetchCNodeUserClockSubquery } = require('./clockManager') +const utils = require('./utils') -describe('Test incrementAndFetchCNodeUserClock', () => { +describe.skip('Test incrementAndFetchCNodeUserClock', () => { const req = { logger: { error: (msg) => console.log(msg) @@ -17,14 +17,17 @@ describe('Test incrementAndFetchCNodeUserClock', () => { const initialClockVal = 0 const incrementClockBy = 1 + let cnodeUser + /** Create cnodeUser */ beforeEach(async () => { const resp = await createStarterCNodeUser() req.session = { cnodeUserUUID: resp.cnodeUserUUID } // Confirm initial clock val in DB - let cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) - assert.strictEqual(cnodeUser.clock, initialClockVal) + cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) + console.log(cnodeUser) + // assert.strictEqual(cnodeUser.clock, initialClockVal) }) /** Wipe all CNodeUsers + dependent data */ @@ -36,6 +39,7 @@ describe('Test incrementAndFetchCNodeUserClock', () => { }) }) + it('Sequential increment&Fetch', async () => { // Explicitly pass in incrementBy value const newClockVal1 = await incrementAndFetchCNodeUserClock(req, incrementClockBy) @@ -58,7 +62,7 @@ describe('Test incrementAndFetchCNodeUserClock', () => { assert.strictEqual(cnodeUser.clock, newClockVal1 + incrementClockBy) }) - it.only('Concurrent increment&Fetch', async () => { + it('Concurrent increment&Fetch', async () => { // Add global sequelize hook to add timeout before cnodeUser.update calls to force concurrent transactions const modelsCopy = models modelsCopy.sequelize.addHook('beforeUpdate', async (instance, options) => { @@ -71,7 +75,7 @@ describe('Test incrementAndFetchCNodeUserClock', () => { proxyquire('./incrementAndFetchCNodeUserClock.js', { '../models': modelsCopy }) // Fire 10 increment&Fetch calls in parallel - const arr = _.range(1,11) // [1,2,3,4,5,6,7,8,9,10] + const arr = _.range(1, 11) // [1,2,3,4,5,6,7,8,9,10] const returnedClockVals = await Promise.all(arr.map(async (i) => { console.log(`calling increment and fetch ${i}...`) return incrementAndFetchCNodeUserClock(req) @@ -79,10 +83,10 @@ describe('Test incrementAndFetchCNodeUserClock', () => { // Ensure returned clock values include no duplicates and include each value from 1-10, in any order const returnedClockValsSorted = returnedClockVals.sort((a, b) => a - b) - assert.deepEqual(returnedClockValsSorted, arr) + assert.deepStrictEqual(returnedClockValsSorted, arr) }) it('Force blocked requests to fail', async () => { - + // return }) }) diff --git a/creator-node/src/fileManager.js b/creator-node/src/fileManager.js index 7ddc388df25..84b129d56c5 100644 --- a/creator-node/src/fileManager.js +++ b/creator-node/src/fileManager.js @@ -13,7 +13,6 @@ const writeFile = promisify(fs.writeFile) const mkdir = promisify(fs.mkdir) const config = require('./config') -const models = require('./models') const Utils = require('./utils') const MAX_AUDIO_FILE_SIZE = parseInt(config.get('maxAudioFileSizeBytes')) // Default = 250,000,000 bytes = 250MB @@ -23,12 +22,11 @@ const ALLOWED_UPLOAD_FILE_EXTENSIONS = config.get('allowedUploadFileExtensions') const AUDIO_MIME_TYPE_REGEX = /audio\/(.*)/ /** - * (1) Add file to IPFS; (2) save file to disk; - * (3) add file via IPFS; (4) save file ref to DB - * @dev - only call this function when file is not already stored to disk - * - if it is, then use saveFileToIPFSFromFS() + * Adds file to IPFS then saves file to disk under multihash name + * + * @dev - only call this function when file is not already stored to disk, else use saveFileToIPFSFromFS() */ -async function saveFileFromBuffer (req, buffer, fileType, clockVal, transaction = null) { +async function saveFileFromBufferToIPFSAndDisk (req, buffer) { // make sure user has authenticated before saving file if (!req.session.cnodeUserUUID) { throw new Error('User must be authenticated to save a file') @@ -36,36 +34,20 @@ async function saveFileFromBuffer (req, buffer, fileType, clockVal, transaction const ipfs = req.app.get('ipfsAPI') + // Add to IPFS without pinning and retrieve multihash const multihash = (await ipfs.add(buffer, { pin: false }))[0].hash + // Write file to disk by multihash for future retrieval const dstPath = path.join(req.app.get('storagePath'), multihash) - await writeFile(dstPath, buffer) - // add reference to file to database - const file = (await models.File.create( - { - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: multihash, - sourceFile: req.fileName, - storagePath: dstPath, - type: fileType, - clock: clockVal - }, - { transaction } - )).dataValues - - req.logger.info('\nAdded file:', multihash, 'file id', file.fileUUID) - return { multihash: multihash, fileUUID: file.fileUUID } + return { multihash, dstPath } } /** - * Save file to IPFS given file path. - * - Add file to IPFS. - * - Re-save file to disk under multihash. - * - Save reference to file in DB. + * Given file path on disk, adds file to IPFS + re-saves under /multihash. */ -async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, clockVal, transaction = null) { +async function saveFileToIPFSFromFS (req, srcPath) { // make sure user has authenticated before saving file if (!req.session.cnodeUserUUID) { throw new Error('User must be authenticated to save a file') @@ -73,35 +55,14 @@ async function saveFileToIPFSFromFS (req, srcPath, fileType, sourceFile, clockVa const ipfs = req.app.get('ipfsAPI') - req.logger.info(`beginning saveFileToIPFSFromFS for srcPath ${srcPath}`) - - let codeBlockTimeStart = Date.now() - + // Add to IPFS without pinning and retrieve multihash const multihash = (await ipfs.addFromFs(srcPath, { pin: false }))[0].hash - req.logger.info(`Time taken in saveFileToIpfsFromFS to add: ${Date.now() - codeBlockTimeStart}`) - codeBlockTimeStart = Date.now() - const dstPath = path.join(req.app.get('storagePath'), multihash) - // store segment file copy under multihash for easy future retrieval + // store file copy by multihash for future retrieval + const dstPath = path.join(req.app.get('storagePath'), multihash) fs.copyFileSync(srcPath, dstPath) - req.logger.info(`Time taken in saveFileToIpfsFromFS to copyFileSync: ${Date.now() - codeBlockTimeStart}`) - - // add reference to file to database - const file = (await models.File.create( - { - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: multihash, - sourceFile: sourceFile, - storagePath: dstPath, - type: fileType, - clock: clockVal - }, - { transaction } - )).dataValues - - req.logger.info(`Added file: ${multihash} for fileUUID ${file.fileUUID} from sourceFile ${sourceFile}`) - return { multihash: multihash, fileUUID: file.fileUUID } + return { multihash, dstPath } } /** @@ -405,7 +366,7 @@ function getFileExtension (fileName) { } module.exports = { - saveFileFromBuffer, + saveFileFromBufferToIPFSAndDisk, saveFileToIPFSFromFS, saveFileForMultihash, removeTrackFolder, diff --git a/creator-node/src/models/clockRecord.js b/creator-node/src/models/clockRecord.js new file mode 100644 index 00000000000..b0772a97f75 --- /dev/null +++ b/creator-node/src/models/clockRecord.js @@ -0,0 +1,49 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + // reference models/File.js + const SourceTableTypesObj = { + AudiusUser: 'AudiusUser', + Track: 'Track', + File: 'File' + } + + const ClockRecord = sequelize.define('ClockRecord', { + cnodeUserUUID: { + type: DataTypes.UUID, + primaryKey: true, // composite PK with clock + unique: false, + allowNull: false, + references: { + model: 'CNodeUsers', + key: 'cnodeUserUUID', + as: 'cnodeUserUUID' + }, + onDelete: 'RESTRICT' + }, + clock: { + type: DataTypes.INTEGER, + primaryKey: true, // composite PK with cnodeUserUUID + unique: false, + allowNull: false + }, + sourceTable: { + // TODO - if this doesn't work, use models/file.js:L46 + type: DataTypes.ENUM( + ...Object.values(SourceTableTypesObj) + ), + allowNull: false + } + }, {}) + + ClockRecord.associate = (models) => { + ClockRecord.belongsTo(models.CNodeUser, { + foreignKey: 'cnodeUserUUID', + sourceKey: 'cnodeUserUUID', + onDelete: 'RESTRICT' + }) + } + + ClockRecord.SourceTableTypesObj = SourceTableTypesObj + + return ClockRecord +} diff --git a/creator-node/src/models/file.js b/creator-node/src/models/file.js index 50300e2f45b..bcc58a3638b 100644 --- a/creator-node/src/models/file.js +++ b/creator-node/src/models/file.js @@ -84,6 +84,7 @@ module.exports = (sequelize, DataTypes) => { }) } + // TODO why no work? File.TrackTypes = ['track', 'copy320'] File.NonTrackTypes = ['dir', 'image', 'metadata'] diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index de14ec487df..74a0b040d5b 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -2,44 +2,60 @@ const { Buffer } = require('ipfs-http-client') const fs = require('fs') const models = require('../models') -const { saveFileFromBuffer } = require('../fileManager') +const { saveFileFromBufferToIPFSAndDisk } = require('../fileManager') const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers') const { getFileUUIDForImageCID } = require('../utils') const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares') -const { incrementAndFetchCNodeUserClock } = require('../utils/incrementAndFetchCNodeUserClock') +const { updateClockInCNodeUserAndClockRecords, selectCNodeUserClockSubquery } = require('../clockManager') const { logger } = require('../logging') module.exports = function (app) { - /** Create AudiusUser from provided metadata, and make metadata available to network. */ + /** + * Create AudiusUser from provided metadata, and make metadata available to network + */ app.post('/audius_users/metadata', authMiddleware, syncLockMiddleware, handleResponse(async (req, res) => { // TODO - input validation const metadataJSON = req.body.metadata - const metadataBuffer = Buffer.from(JSON.stringify(metadataJSON)) + const cnodeUserUUID = req.session.cnodeUserUUID + // Save file from buffer to IPFS and disk + // TODO simplify + let multihash, dstPath + try { + const resp = await saveFileFromBufferToIPFSAndDisk(req, metadataBuffer) + multihash = resp.multihash + dstPath = resp.dstPath + } catch (e) { + return errorResponseServerError(`saveFileFromBufferToIPFSAndDisk op failed: ${e}`) + } + + // Record metadata file entry in DB const transaction = await models.sequelize.transaction() - let multihash, fileUUID + let fileUUID try { - // increment and fetch cnodeUser.clock value - const newClockVal = await incrementAndFetchCNodeUserClock(req) - - const saveFileFromBufferResp = await saveFileFromBuffer( - req, - metadataBuffer, - 'metadata', - newClockVal, - transaction - ) - multihash = saveFileFromBufferResp.multihash - fileUUID = saveFileFromBufferResp.fileUUID + await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) + + fileUUID = (await models.File.create({ + cnodeUserUUID, + multihash, + sourceFile: req.fileName, + storagePath: dstPath, + type: 'metadata', // TODO - replace with models enum + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + }, { transaction }) + ).dataValues.fileUUID await transaction.commit() } catch (e) { await transaction.rollback() - return errorResponseServerError(`Could not save file to disk, ipfs, and/or db: ${e}`) + return errorResponseServerError(`Could not save to db: ${e}`) } - return successResponse({ 'metadataMultihash': multihash, 'metadataFileUUID': fileUUID }) + return successResponse({ + 'metadataMultihash': multihash, + 'metadataFileUUID': fileUUID + }) })) /** @@ -63,7 +79,7 @@ module.exports = function (app) { // Fetch metadataJSON for metadataFileUUID. const file = await models.File.findOne({ where: { fileUUID: metadataFileUUID, cnodeUserUUID } }) if (!file) { - return errorResponseBadRequest(`No file found for provided metadataFileUUID ${metadataFileUUID}.`) + return errorResponseBadRequest(`No file db record found for provided metadataFileUUID ${metadataFileUUID}.`) } let metadataJSON try { @@ -82,11 +98,11 @@ module.exports = function (app) { } const transaction = await models.sequelize.transaction() + try { logger.info(`beginning audiusUsers DB transactions`) - // increment and fetch cnodeUser.clock value - const newClockVal = await incrementAndFetchCNodeUserClock(req) + await updateClockInCNodeUserAndClockRecords(req, 'AudiusUser', transaction) // Insert new audiusUser entry to DB const audiusUser = await models.AudiusUser.create({ @@ -96,7 +112,7 @@ module.exports = function (app) { blockchainId: blockchainUserId, coverArtFileUUID, profilePicFileUUID, - clock: newClockVal + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) }, { transaction, returning: true }) // Update cnodeUser's latestBlockNumber and clock diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js index 51dec5e8b84..2f17b2cf9b1 100644 --- a/creator-node/src/routes/files.js +++ b/creator-node/src/routes/files.js @@ -22,7 +22,7 @@ const { authMiddleware, syncLockMiddleware, triggerSecondarySyncs } = require('. const { getIPFSPeerId, ipfsSingleByteCat, ipfsStat } = require('../utils') const ImageProcessingQueue = require('../ImageProcessingQueue') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') -const { incrementAndFetchCNodeUserClock } = require('../utils/incrementAndFetchCNodeUserClock') +const { updateClockInCNodeUserAndClockRecords, selectCNodeUserClockSubquery } = require('../clockManager') /** * Helper method to stream file from file system on creator node @@ -235,7 +235,9 @@ const getDirCID = async (req, res) => { } module.exports = function (app) { - /** Store image in multiple-resolutions on disk + DB and make available via IPFS */ + /** + * Store image in multiple-resolutions on disk + DB and make available via IPFS + */ app.post('/image_upload', authMiddleware, syncLockMiddleware, uploadTempDiskStorage.single('file'), handleResponse(async (req, res) => { if (!req.body.square || !(req.body.square === 'true' || req.body.square === 'false')) { return errorResponseBadRequest('Must provide square boolean param in request body') @@ -243,13 +245,14 @@ module.exports = function (app) { if (!req.file) { return errorResponseBadRequest('Must provide image file in request body.') } - let routestart = Date.now() + const routestart = Date.now() const imageBufferOriginal = req.file.path const originalFileName = req.file.originalname - let resizeResp + const cnodeUserUUID = req.session.cnodeUserUUID // Resize the images and add them to IPFS and filestorage + let resizeResp try { if (req.body.square === 'true') { resizeResp = await ImageProcessingQueue.resizeImage({ @@ -278,58 +281,47 @@ module.exports = function (app) { }) } - req.logger.info('ipfs add resp', resizeResp) + req.logger.debug('ipfs add resp', resizeResp) } catch (e) { return errorResponseServerError(e) } - const t = await models.sequelize.transaction() - // Add the created files to the DB + // Record image file entries in DB + const transaction = await models.sequelize.transaction() try { - // increment and fetch cnodeUser.clock value - const finalClockVal = await incrementAndFetchCNodeUserClock(req, resizeResp.files.length + 1) - const initialClockVal = finalClockVal - (resizeResp.files.length + 1) - - // Save dir file reference to DB - const dir = (await models.File.create( - { - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: resizeResp.dir.dirCID, - sourceFile: null, - storagePath: resizeResp.dir.dirDestPath, - type: 'dir', - clock: (initialClockVal + 1) - }, - { transaction: t } - )).dataValues - - // Save each file to the DB - await Promise.all(resizeResp.files.map(async (fileResp, i) => { - const file = (await models.File.create( - { - cnodeUserUUID: req.session.cnodeUserUUID, - multihash: fileResp.multihash, - sourceFile: fileResp.sourceFile, - storagePath: fileResp.storagePath, - type: 'image', - dirMultihash: resizeResp.dir.dirCID, - fileName: fileResp.sourceFile.split('/').slice(-1)[0], - clock: (initialClockVal + 1 + (i + 1)) // increment clock val for each file entry - }, - { transaction: t } - )).dataValues - - req.logger.info('Added file', fileResp, file) - })) + // Record dir file entry in DB + await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) + await models.File.create({ + cnodeUserUUID, + multihash: resizeResp.dir.dirCID, + sourceFile: null, + storagePath: resizeResp.dir.dirDestPath, + type: 'dir', // TODO - replace with models enum + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + }, { transaction }) + + // Record all image res file entries in DB + // Must be written sequentially to ensure clock values are correctly incremented and populated + for (const file of resizeResp.files) { + await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) + await models.File.create({ + cnodeUserUUID, + multihash: file.multihash, + sourceFile: file.sourceFile, + storagePath: file.storagePath, + type: 'image', // TODO - replace with models enum + dirMultihash: resizeResp.dir.dirCID, + fileName: file.sourceFile.split('/').slice(-1)[0], + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + }) + } - req.logger.info('Added all files for dir', dir) req.logger.info(`route time = ${Date.now() - routestart}`) - - await t.commit() + await transaction.commit() triggerSecondarySyncs(req) return successResponse({ dirCID: resizeResp.dir.dirCID }) } catch (e) { - await t.rollback() + await transaction.rollback() return errorResponseServerError(e) } })) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 1eefc633811..10fd2fe5c40 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -5,7 +5,7 @@ const { Buffer } = require('ipfs-http-client') const config = require('../config.js') const { getSegmentsDuration } = require('../segmentDuration') const models = require('../models') -const { saveFileFromBuffer, saveFileToIPFSFromFS, removeTrackFolder, handleTrackContentUpload } = require('../fileManager') +const { saveFileFromBufferToIPFSAndDisk, saveFileToIPFSFromFS, removeTrackFolder, handleTrackContentUpload } = require('../fileManager') const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError, errorResponseForbidden } = require('../apiHelpers') const { getFileUUIDForImageCID } = require('../utils') const { authMiddleware, ensurePrimaryMiddleware, syncLockMiddleware, triggerSecondarySyncs } = require('../middlewares') @@ -14,13 +14,11 @@ const { getCID } = require('./files') const { decode } = require('../hashids.js') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') const { logger } = require('../logging.js') -const { incrementAndFetchCNodeUserClock } = require('../utils/incrementAndFetchCNodeUserClock') +const { updateClockInCNodeUserAndClockRecords, selectCNodeUserClockSubquery } = require('../clockManager') module.exports = function (app) { /** * upload track segment files and make avail - will later be associated with Audius track - * @dev - currently stores each segment twice, once under random file UUID & once under IPFS multihash - * - this should be addressed eventually * @dev - Prune upload artifacts after successful and failed uploads. Make call without awaiting, and let async queue clean up. */ app.post('/track_content', authMiddleware, ensurePrimaryMiddleware, syncLockMiddleware, handleTrackContentUpload, handleResponse(async (req, res) => { @@ -36,13 +34,17 @@ module.exports = function (app) { return errorResponseBadRequest(req.fileFilterError) } + const routeTimeStart = Date.now() - let codeBlockTimeStart = Date.now() + let codeBlockTimeStart + const cnodeUserUUID = req.session.cnodeUserUUID - // create and save track file transcoded version and segments to disk + // Create track transcode and segments, and save all to disk let transcodedFilePath let segmentFilePaths try { + codeBlockTimeStart = Date.now() + const transcode = await Promise.all([ TranscodingQueue.segment(req.fileDir, req.fileName, { logContext: req.logContext }), TranscodingQueue.transcode320(req.fileDir, req.fileName, { logContext: req.logContext }) @@ -50,7 +52,7 @@ module.exports = function (app) { segmentFilePaths = transcode[0].filePaths transcodedFilePath = transcode[1].filePath - req.logger.info(`Time taken in /track_content to re-encode track file: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + logger.info(`Time taken in /track_content to re-encode track file: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) } catch (err) { // Prune upload artifacts removeTrackFolder(req, req.fileDir) @@ -58,78 +60,27 @@ module.exports = function (app) { return errorResponseServerError(err) } - // for each path, call saveFile and get back multihash; return multihash + segment duration - // run all async ops in parallel as they are independent + // Save transcode and segment files (in parallel) to ipfs and retrieve multihashes codeBlockTimeStart = Date.now() - const t = await models.sequelize.transaction() - - let transcodedFilePromResp - let segmentSaveFilePromResps - let segmentDurations - try { - // increment and fetch cnodeUser.clock value - const finalClockVal = await incrementAndFetchCNodeUserClock(req, segmentFilePaths.length + 1) - const initialClockVal = finalClockVal - (segmentFilePaths.length + 1) - - // Call saveFileToIPFSFromFS for transcode file - transcodedFilePromResp = await saveFileToIPFSFromFS( - req, - transcodedFilePath, - 'copy320', - req.fileName, - (initialClockVal + 1), - t - ) - - // Call saveFileToIPFSFromFS for each track segment file - req.logger.info(`segmentFilePaths.length ${segmentFilePaths.length}`) - segmentSaveFilePromResps = await Promise.all(segmentFilePaths.map(async (filePath, i) => { - const absolutePath = path.join(req.fileDir, 'segments', filePath) - req.logger.info(`about to perform saveFileToIPFSFromFS #${i}`) - let response = await saveFileToIPFSFromFS( - req, - absolutePath, - 'track', - req.fileName, - (initialClockVal + 1 + (i + 1)), - t - ) - response.segmentName = filePath - return response - })) - - req.logger.info(`Time taken in /track_content for saving segments and transcoding to IPFS: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) - - codeBlockTimeStart = Date.now() - let fileSegmentPath = path.join(req.fileDir, 'segments') - segmentDurations = await getSegmentsDuration( - req, - fileSegmentPath, - req.fileName, - req.file.destination - ) - req.logger.info(`Time taken in /track_content to get segment duration: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) - - // Commit transaction - codeBlockTimeStart = Date.now() - req.logger.info(`attempting to commit tx for file ${req.fileName}`) - await t.commit() - } catch (e) { - req.logger.info(`failed to commit...rolling back. file ${req.fileName}`) - - await t.rollback() - - // Prune upload artifacts - removeTrackFolder(req, req.fileDir) - - return errorResponseServerError(e) - } - req.logger.info(`Time taken in /track_content to commit tx block to db: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) - - let trackSegments = segmentSaveFilePromResps.map((saveFileResp, i) => { - let segmentName = saveFileResp.segmentName - let duration = segmentDurations[segmentName] - return { 'multihash': saveFileResp.multihash, 'duration': duration } + const transcodeFileIPFSResp = await saveFileToIPFSFromFS(req, transcodedFilePath) + const segmentFileIPFSResps = await Promise.all(segmentFilePaths.map(async (segmentFilePath) => { + const segmentAbsolutePath = path.join(req.fileDir, 'segments', segmentFilePath) + const { multihash, dstPath } = await saveFileToIPFSFromFS(req, segmentAbsolutePath) + return { multihash, srcPath: segmentFilePath, dstPath } + })) + logger.info(`Time taken in /track_content for saving transcode + segment files to IPFS: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + + // Retrieve all segment durations as map(segment srcFilePath => segment duration) + codeBlockTimeStart = Date.now() + const segmentDurations = await getSegmentsDuration(req.fileName, req.file.destination) + logger.info(`Time taken in /track_content to get segment duration: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + + // For all segments, build array of (segment multihash, segment duration) + let trackSegments = segmentFileIPFSResps.map((segmentFileIPFSResp) => { + return { + multihash: segmentFileIPFSResp.multihash, + duration: segmentDurations[segmentFileIPFSResp.srcPath] + } }) // exclude 0-length segments that are sometimes outputted by ffmpeg segmentation @@ -143,7 +94,7 @@ module.exports = function (app) { return errorResponseServerError('Track upload failed - no track segments') } - // Don't allow if any segment CID is in blacklist. + // Error if any segment CID is in blacklist. try { await Promise.all(trackSegments.map(async segmentObj => { if (await req.app.get('blacklistManager').CIDIsInBlacklist(segmentObj.multihash)) { @@ -161,13 +112,55 @@ module.exports = function (app) { } } - // Prune upload artifacts + // Record entries for transcode and segment files in DB + codeBlockTimeStart = Date.now() + const transaction = await models.sequelize.transaction() + let transcodeFileUUID + try { + // Record transcode file entry in DB + await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) + transcodeFileUUID = (await models.File.create({ + cnodeUserUUID, + multihash: transcodeFileIPFSResp.multihash, + sourceFile: req.fileName, + storagePath: transcodeFileIPFSResp.dstPath, + type: 'copy320', // TODO - replace with models enum + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + }, { transaction }) + ).dataValues.fileUUID + + // Record all segment file entries in DB + // Must be written sequentially to ensure clock values are correctly incremented and populated + for (const { multihash, dstPath } of segmentFileIPFSResps) { + await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) + await models.File.create({ + cnodeUserUUID, + multihash, + sourceFile: req.fileName, + storagePath: dstPath, + type: 'track', // TODO - replace with models enum + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + }, { transaction }) + } + + await transaction.commit() + } catch (e) { + await transaction.rollback() + + // Prune upload artifacts + removeTrackFolder(req, req.fileDir) + + return errorResponseServerError(e) + } + logger.info(`Time taken in /track_content for DB updates: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + + // Prune upload artifacts after success removeTrackFolder(req, req.fileDir) - req.logger.info(`Time taken in /track_content for full route: ${Date.now() - routeTimeStart}ms for file ${req.fileName}`) + logger.info(`Time taken in /track_content for full route: ${Date.now() - routeTimeStart}ms for file ${req.fileName}`) return successResponse({ - 'transcodedTrackCID': transcodedFilePromResp.multihash, - 'transcodedTrackUUID': transcodedFilePromResp.fileUUID, + 'transcodedTrackCID': transcodeFileIPFSResp.multihash, + 'transcodedTrackUUID': transcodeFileUUID, 'track_segments': trackSegments, 'source_file': req.fileName }) @@ -234,29 +227,40 @@ module.exports = function (app) { } } - // Store + pin metadata multihash to disk + IPFS. const metadataBuffer = Buffer.from(JSON.stringify(metadataJSON)) + const cnodeUserUUID = req.session.cnodeUserUUID + // Save file from buffer to IPFS and disk + // TODO simplify + let multihash, dstPath + try { + const resp = await saveFileFromBufferToIPFSAndDisk(req, metadataBuffer) + multihash = resp.multihash + dstPath = resp.dstPath + } catch (e) { + return errorResponseServerError(`/tracks/metadata saveFileFromBufferToIPFSAndDisk op failed: ${e}`) + } + + // Record metadata file entry in DB const transaction = await models.sequelize.transaction() - let multihash, fileUUID + let fileUUID try { - // increment and fetch cnodeUser.clock value - const newClockVal = await incrementAndFetchCNodeUserClock(req) - - const saveFileFromBufferResp = await saveFileFromBuffer( - req, - metadataBuffer, - 'metadata', - newClockVal, - transaction - ) - multihash = saveFileFromBufferResp.multihash - fileUUID = saveFileFromBufferResp.fileUUID + await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) + + fileUUID = (await models.File.create({ + cnodeUserUUID, + multihash, + sourceFile: req.fileName, + storagePath: dstPath, + type: 'metadata', // TODO - replace with models enum + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + }, { transaction }) + ).dataValues.fileUUID await transaction.commit() } catch (e) { await transaction.rollback() - return errorResponseServerError(`Could not save file to disk, ipfs, and/or db: ${e}`) + return errorResponseServerError(`Could not save to db db: ${e}`) } return successResponse({ @@ -272,36 +276,39 @@ module.exports = function (app) { app.post('/tracks', authMiddleware, ensurePrimaryMiddleware, syncLockMiddleware, handleResponse(async (req, res) => { const { blockchainTrackId, blockNumber, metadataFileUUID, transcodedTrackUUID } = req.body + // Input validation if (!blockchainTrackId || !blockNumber || !metadataFileUUID) { return errorResponseBadRequest('Must include blockchainTrackId, blockNumber, and metadataFileUUID.') } - // Error on outdated blocknumber. + // Error on outdated blocknumber const cnodeUser = req.session.cnodeUser if (!cnodeUser.latestBlockNumber || cnodeUser.latestBlockNumber > blockNumber) { return errorResponseBadRequest(`Invalid blockNumber param. Must be higher than previously processed blocknumber.`) } const cnodeUserUUID = req.session.cnodeUserUUID - // Fetch metadataJSON for metadataFileUUID. + // Fetch metadataJSON for metadataFileUUID, error if not found or malformatted const file = await models.File.findOne({ where: { fileUUID: metadataFileUUID, cnodeUserUUID } }) if (!file) { - return errorResponseBadRequest(`No file found for provided metadataFileUUID ${metadataFileUUID}.`) + return errorResponseBadRequest(`No file db record found for provided metadataFileUUID ${metadataFileUUID}.`) } let metadataJSON try { metadataJSON = JSON.parse(fs.readFileSync(file.storagePath)) - if (!metadataJSON || - !metadataJSON.track_segments || - !Array.isArray(metadataJSON.track_segments) || - !metadataJSON.track_segments.length) { + if ( + !metadataJSON || + !metadataJSON.track_segments || + !Array.isArray(metadataJSON.track_segments) || + !metadataJSON.track_segments.length + ) { return errorResponseServerError(`Malformatted metadataJSON stored for metadataFileUUID ${metadataFileUUID}.`) } } catch (e) { return errorResponseServerError(`No file stored on disk for metadataFileUUID ${metadataFileUUID} at storagePath ${file.storagePath}.`) } - // Get coverArtFileUUID for multihash in metadata object, if present. + // Get coverArtFileUUID for multihash in metadata object, else error let coverArtFileUUID try { coverArtFileUUID = await getFileUUIDForImageCID(req, metadataJSON.cover_art_sizes) @@ -309,11 +316,9 @@ module.exports = function (app) { return errorResponseServerError(e.message) } - const t = await models.sequelize.transaction() - + logger.debug('Beginning POST /tracks DB transactions') + const transaction = await models.sequelize.transaction() try { - logger.debug('Beginning POST /tracks DB transactions') - const existingTrackEntry = await models.Track.findOne({ where: { cnodeUserUUID, @@ -321,11 +326,10 @@ module.exports = function (app) { blockchainId: blockchainTrackId, coverArtFileUUID }, - transaction: t + transaction }) - // increment and fetch cnodeUser.clock value - const newClockVal = await incrementAndFetchCNodeUserClock(req) + await updateClockInCNodeUserAndClockRecords(req, 'Track', transaction) // Insert new track entry on db (for track update, a new entry is still created with incremented clock val) const track = await models.Track.create({ @@ -334,17 +338,22 @@ module.exports = function (app) { metadataJSON, blockchainId: blockchainTrackId, coverArtFileUUID, - clock: newClockVal - }, { transaction: t } + clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + }, { transaction } ) - /** Associate matching segment files on DB with new/updated track. */ + /** + * Associate matching transcode & segment files on DB with new/updated track + * Must be done in same transaction to atomicity + * + * TODO - consider implications of edge-case -> two attempted /track_content before associate + */ const trackSegmentCIDs = metadataJSON.track_segments.map(segment => segment.multihash) - // if track created, ensure files exist with trackuuid = null and update them. + // if track created, ensure files exist with trackuuid = null and update them if (!existingTrackEntry) { - // Update the transcoded 320kbps copy + // Associate the transcode file db record with trackUUID if (transcodedTrackUUID) { const transcodedFile = await models.File.findOne({ where: { @@ -353,7 +362,7 @@ module.exports = function (app) { trackUUID: null, type: 'copy320' }, - transaction: t + transaction }) if (!transcodedFile) { throw new Error('Did not find a transcoded file for the provided CID.') @@ -364,9 +373,9 @@ module.exports = function (app) { fileUUID: transcodedTrackUUID, cnodeUserUUID, trackUUID: null, - type: 'copy320' + type: 'copy320' // TODO - replace with model enum }, - transaction: t + transaction } ) if (numAffectedRows === 0) { @@ -374,7 +383,7 @@ module.exports = function (app) { } } - // Update the corresponding segment files + // Associate all segment file db records with trackUUID const trackFiles = await models.File.findAll({ where: { multihash: trackSegmentCIDs, @@ -382,9 +391,10 @@ module.exports = function (app) { trackUUID: null, type: 'track' }, - transaction: t + transaction }) - if (trackFiles.length < trackSegmentCIDs.length) { + + if (trackFiles.length !== trackSegmentCIDs.length) { throw new Error('Did not find files for every track segment CID.') } const numAffectedRows = await models.File.update( @@ -395,14 +405,15 @@ module.exports = function (app) { trackUUID: null, type: 'track' }, - transaction: t + transaction } ) - if (numAffectedRows < trackSegmentCIDs.length) { + if (numAffectedRows !== trackSegmentCIDs.length) { + logger.error(`\n\n\nnumAffectedRows: ${numAffectedRows}`) throw new Error('Failed to associate files for every track segment CID.') } } else { /** If track updated, ensure files exist with trackuuid. */ - // Check the transcoded copy if present + // Ensure transcode file db record exists if uuid provided if (transcodedTrackUUID) { const transcodedFile = await models.File.findOne({ where: { @@ -411,14 +422,14 @@ module.exports = function (app) { trackUUID: track.trackUUID, type: 'copy320' }, - transaction: t + transaction }) if (!transcodedFile) { throw new Error('Did not find the corresponding transcoded file for the provided track UUID.') } } - // Check the segment files + // Ensure segment file db records exist for all CIDs const trackFiles = await models.File.findAll({ where: { multihash: trackSegmentCIDs, @@ -426,7 +437,7 @@ module.exports = function (app) { trackUUID: track.trackUUID, type: 'track' }, - transaction: t + transaction }) if (trackFiles.length < trackSegmentCIDs.length) { throw new Error('Did not find files for every track segment CID with trackUUID.') @@ -435,28 +446,28 @@ module.exports = function (app) { // Update cnodeUser's latestBlockNumber if higher than previous latestBlockNumber. // TODO - move to subquery to guarantee atomicity. - const updatedCNodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID }, transaction: t }) + // TODO - can deprecate with clockwork? + const updatedCNodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID }, transaction }) if (!updatedCNodeUser || !updatedCNodeUser.latestBlockNumber) { throw new Error('Issue in retrieving udpatedCnodeUser') } - req.logger.info( + logger.info( `cnodeuser ${cnodeUserUUID} first latestBlockNumber ${cnodeUser.latestBlockNumber} || \ current latestBlockNumber ${updatedCNodeUser.latestBlockNumber} || \ given blockNumber ${blockNumber}` ) if (blockNumber > updatedCNodeUser.latestBlockNumber) { // Update cnodeUser's latestBlockNumber - await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction: t }) + await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction }) } logger.info(`completed POST tracks route`) - - await t.commit() + await transaction.commit() triggerSecondarySyncs(req) return successResponse({ trackUUID: track.trackUUID }) } catch (e) { - req.logger.error(e.message) - await t.rollback() + logger.error(e.message) + await transaction.rollback() return errorResponseServerError(e.message) } })) @@ -544,7 +555,7 @@ module.exports = function (app) { } if (libs.identityService) { - req.logger.info(`Logging listen for track ${blockchainId} by ${delegateOwnerWallet}`) + logger.info(`Logging listen for track ${blockchainId} by ${delegateOwnerWallet}`) // Fire and forget listen recording // TODO: Consider queueing these requests libs.identityService.logTrackListen(blockchainId, delegateOwnerWallet) diff --git a/creator-node/src/routes/users.js b/creator-node/src/routes/users.js index d4ee8abadd2..7e6b376b3dd 100644 --- a/creator-node/src/routes/users.js +++ b/creator-node/src/routes/users.js @@ -35,11 +35,12 @@ module.exports = function (app) { return successResponse() // do nothing if user already exists } + // Create CNodeUser entry for wallet with clock = 0 await models.CNodeUser.create({ walletPublicKey: walletAddress, - // Initialize clock value for cnodeUser to 0 clock: 0 }) + return successResponse() })) diff --git a/creator-node/src/segmentDuration.js b/creator-node/src/segmentDuration.js index fcb60e8fb47..deb10c16f58 100644 --- a/creator-node/src/segmentDuration.js +++ b/creator-node/src/segmentDuration.js @@ -3,8 +3,8 @@ const fs = require('fs') const SEGMENT_REGEXP = /(segment[0-9]*.ts)/ -// Parse m3u8 file from HLS output and return mapped segment durations -async function getSegmentsDuration (req, segmentPath, filename, filedir) { +// Parse m3u8 file from HLS output and return map(segment filePath (segmentName) => segment duration) +async function getSegmentsDuration (filename, filedir) { return new Promise((resolve, reject) => { try { let splitResults = filename.split('.') diff --git a/creator-node/src/utils/incrementAndFetchCNodeUserClock.js b/creator-node/src/utils/incrementAndFetchCNodeUserClock.js deleted file mode 100644 index 366be675fa5..00000000000 --- a/creator-node/src/utils/incrementAndFetchCNodeUserClock.js +++ /dev/null @@ -1,32 +0,0 @@ -const models = require('../models') - -// TODO consider adding hook to ensure write op can never set clockVal to anything <= current -const incrementAndFetchCNodeUserClock = async (req, incrementBy = 1) => { - const transaction = await models.sequelize.transaction() - - try { - const cnodeUser = await models.CNodeUser.findOne({ - where: { cnodeUserUUID: req.session.cnodeUserUUID }, - transaction, - /** TODO add comment */ - lock: transaction.LOCK.UPDATE - }) - - const newClockVal = (cnodeUser.clock + incrementBy) - - await cnodeUser.update( - { clock: newClockVal }, - { transaction } - ) - - await transaction.commit() - return newClockVal - } catch (e) { - await transaction.rollback() - throw new Error('Failed to increment cnodeUser.clock') - } -} - -module.exports = { - incrementAndFetchCNodeUserClock -} diff --git a/creator-node/test/audiusUsers.js b/creator-node/test/audiusUsers.test.js similarity index 93% rename from creator-node/test/audiusUsers.js rename to creator-node/test/audiusUsers.test.js index 68ec5dedcbd..274d80696b0 100644 --- a/creator-node/test/audiusUsers.js +++ b/creator-node/test/audiusUsers.test.js @@ -5,6 +5,7 @@ const path = require('path') const fs = require('fs') const models = require('../src/models') + const ipfsClient = require('../src/ipfsClient') const config = require('../src/config') const BlacklistManager = require('../src/blacklistManager') @@ -52,7 +53,7 @@ describe('test AudiusUsers', function () { const resp = await request(app) .post('/audius_users/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata }) .expect(200) @@ -70,7 +71,7 @@ describe('test AudiusUsers', function () { const resp = await request(app) .post('/audius_users/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata }) .expect(200) @@ -80,7 +81,7 @@ describe('test AudiusUsers', function () { await request(app) .post('/audius_users') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ blockchainUserId: 1, blockNumber: 10, metadataFileUUID: resp.body.metadataFileUUID }) .expect(200) }) @@ -122,7 +123,7 @@ describe('tests /audius_users/metadata metadata upload with actual ipfsClient fo it('should fail if metadata is not found in request body', async function () { const resp = await request(app) .post('/audius_users/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ dummy: 'data' }) .expect(500) @@ -136,18 +137,18 @@ describe('tests /audius_users/metadata metadata upload with actual ipfsClient fo const metadata = { metadata: 'spaghetti' } const resp = await request(app) .post('/audius_users/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send(metadata) .expect(500) - assert.deepStrictEqual(resp.body.error, 'Could not save file to disk, ipfs, and/or db: Error: ipfs add failed!') + assert.deepStrictEqual(resp.body.error, 'saveFileFromBufferToIPFSAndDisk op failed: Error: ipfs add failed!') }) it('should successfully add metadata file to filesystem, db, and ipfs', async function () { const metadata = sortKeys({ spaghetti: 'spaghetti' }) const resp = await request(app) .post('/audius_users/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata }) .expect(200) diff --git a/creator-node/test/expressApp.js b/creator-node/test/expressApp.test.js similarity index 96% rename from creator-node/test/expressApp.js rename to creator-node/test/expressApp.test.js index f91b9839967..6c46fa4f5b1 100644 --- a/creator-node/test/expressApp.js +++ b/creator-node/test/expressApp.test.js @@ -44,7 +44,7 @@ describe('test expressApp', function () { // logout endpoint requires login / checks session request(app) .post('/users/logout') - .set('X-Session-ID', session + '1') + .set('X-Session-ID', session.sessionToken + '1') .expect(401, done) }) diff --git a/creator-node/test/ffmpeg.js b/creator-node/test/ffmpeg.test.js similarity index 97% rename from creator-node/test/ffmpeg.js rename to creator-node/test/ffmpeg.test.js index 29d96fc3230..16cc2558fed 100644 --- a/creator-node/test/ffmpeg.js +++ b/creator-node/test/ffmpeg.test.js @@ -29,7 +29,6 @@ describe('test segmentFile()', () => { await segmentFile(null, null, {}) assert.fail('Should have thrown error with bad params') } catch (e) { - console.error(e) assert.ok(e.message) } }) @@ -47,7 +46,6 @@ describe('test segmentFile()', () => { await segmentFile(fileDir, fileName, {}) assert.fail('Should have thrown error when segmenting a bad track (image)') } catch (e) { - console.error(e) assert.deepStrictEqual(e.message, 'FFMPEG Error') } }) @@ -64,7 +62,6 @@ describe('test segmentFile()', () => { try { await segmentFile(fileDir, fileName, {}) } catch (e) { - console.error(e) assert.fail(e.message) } diff --git a/creator-node/test/fileManager.js b/creator-node/test/fileManager.test.js similarity index 81% rename from creator-node/test/fileManager.js rename to creator-node/test/fileManager.test.js index 5511ab9bf41..51c2d2924b3 100644 --- a/creator-node/test/fileManager.js +++ b/creator-node/test/fileManager.test.js @@ -6,7 +6,7 @@ const fsExtra = require('fs-extra') const path = require('path') const { ipfs } = require('../src/ipfsClient') -const { saveFileToIPFSFromFS, removeTrackFolder, saveFileFromBuffer } = require('../src/fileManager') +const { saveFileToIPFSFromFS, removeTrackFolder, saveFileFromBufferToIPFSAndDisk } = require('../src/fileManager') const config = require('../src/config') const models = require('../src/models') @@ -39,16 +39,14 @@ const req = { const segmentsDirPath = 'test/test-segments' const sourceFile = 'segment001.ts' const srcPath = path.join(segmentsDirPath, sourceFile) -const fileType = 'track' -// consts used for testing saveFileFromBuffer() +// consts used for testing saveFileFromBufferToIPFSAndDisk() const metadata = { test: 'field1', track_segments: [{ 'multihash': 'testCIDLink', 'duration': 1000 }], owner_id: 1 } const buffer = Buffer.from(JSON.stringify(metadata)) -const clockVal = 1 describe('test fileManager', () => { afterEach(function () { @@ -74,7 +72,7 @@ describe('test fileManager', () => { } try { - await saveFileToIPFSFromFS(reqOverride, srcPath, fileType, sourceFile, clockVal) + await saveFileToIPFSFromFS(reqOverride, srcPath) assert.fail('Should not have passed if cnodeUserUUID is not present in request.') } catch (e) { assert.deepStrictEqual(e.message, 'User must be authenticated to save a file') @@ -90,7 +88,7 @@ describe('test fileManager', () => { sinon.stub(ipfs, 'addFromFs').rejects(new Error('ipfs is down!')) try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) + await saveFileToIPFSFromFS(req, srcPath) assert.fail('Should not have passed if ipfs is down.') } catch (e) { assert.deepStrictEqual(e.message, 'ipfs is down!') @@ -106,29 +104,13 @@ describe('test fileManager', () => { sinon.stub(fs, 'copyFileSync').throws(new Error('Failed to copy files!!')) try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) + await saveFileToIPFSFromFS(req, srcPath) assert.fail('Should not have passed if file copying fails.') } catch (e) { assert.deepStrictEqual(e.message, 'Failed to copy files!!') } }) - /** - * Given: a file is being saved to ipfs from fs - * When: the db connection is down - * Then: an error is thrown - */ - it('should throw an error if db connection is down', async () => { - sinon.stub(models.File, 'create').rejects(new Error('Failed to find or create file!!!')) - - try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) - assert.fail('Should not have passed if db connection is down.') - } catch (e) { - assert.deepStrictEqual(e.message, 'Failed to find or create file!!!') - } - }) - /** * Given: a file is being saved to ipfs from fs * When: everything works as expected @@ -141,7 +123,7 @@ describe('test fileManager', () => { sinon.stub(models.File, 'create').returns({ dataValues: { fileUUID: 'uuid' } }) try { - await saveFileToIPFSFromFS(req, srcPath, fileType, sourceFile, clockVal) + await saveFileToIPFSFromFS(req, srcPath) } catch (e) { assert.fail(e.message) } @@ -169,8 +151,8 @@ describe('test fileManager', () => { }) }) - // ~~~~~~~~~~~~~~~~~~~~~~~~~ saveFileFromBuffer() TESTS ~~~~~~~~~~~~~~~~~~~~~~~~~ - describe('test saveFileFromBuffer()', () => { + // ~~~~~~~~~~~~~~~~~~~~~~~~~ saveFileFromBufferToIPFSAndDisk() TESTS ~~~~~~~~~~~~~~~~~~~~~~~~~ + describe('test saveFileFromBufferToIPFSAndDisk()', () => { /** * Given: a file buffer is being saved to ipfs, fs, and db * When: cnodeUserUUID is not present @@ -188,7 +170,7 @@ describe('test fileManager', () => { } try { - await saveFileFromBuffer(reqOverride, buffer, 'metadata') + await saveFileFromBufferToIPFSAndDisk(reqOverride, buffer) assert.fail('Should not have passed if cnodeUserUUID is not present in request.') } catch (e) { assert.deepStrictEqual(e.message, 'User must be authenticated to save a file') @@ -204,7 +186,7 @@ describe('test fileManager', () => { sinon.stub(ipfs, 'add').rejects(new Error('ipfs is down!')) try { - await saveFileFromBuffer(req, buffer, 'metadata') + await saveFileFromBufferToIPFSAndDisk(req, buffer) assert.fail('Should not have passed if ipfs is down.') } catch (e) { assert.deepStrictEqual(e.message, 'ipfs is down!') @@ -220,29 +202,13 @@ describe('test fileManager', () => { sinon.stub(ipfs, 'add').resolves([{ hash: 'bad/path/fail' }]) // pass bad data to writeFile() try { - await saveFileFromBuffer(req, buffer, 'metadata') + await saveFileFromBufferToIPFSAndDisk(req, buffer) assert.fail('Should not have passed if writing to filesystem fails.') } catch (e) { assert.ok(e.message) } }) - /** - * Given: a file buffer is being saved to ipfs, fs, and db - * When: adding reference to db fails - * Then: an error is thrown - */ - it('should throw an error if writing reference to db fails', async () => { - sinon.stub(models.File, 'create').rejects(new Error('Failed to find or create file!!!')) - - try { - await saveFileFromBuffer(req, buffer, 'metadata', clockVal) - assert.fail('Should not have if db connection is down.') - } catch (e) { - assert.deepStrictEqual(e.message, 'Failed to find or create file!!!') - } - }) - /** * Given: a file buffer is being saved to ipfs, fs, and db * When: everything works as expected @@ -253,7 +219,7 @@ describe('test fileManager', () => { let resp try { - resp = await saveFileFromBuffer(req, buffer, 'metadata', clockVal) + resp = await saveFileFromBufferToIPFSAndDisk(req, buffer) } catch (e) { assert.fail(e.message) } diff --git a/creator-node/test/hashids.js b/creator-node/test/hashids.test.js similarity index 100% rename from creator-node/test/hashids.js rename to creator-node/test/hashids.test.js diff --git a/creator-node/test/resizeImage.js b/creator-node/test/resizeImage.test.js similarity index 100% rename from creator-node/test/resizeImage.js rename to creator-node/test/resizeImage.test.js diff --git a/creator-node/test/tracks.js b/creator-node/test/tracks.js index c594b6fba82..a4ece23782a 100644 --- a/creator-node/test/tracks.js +++ b/creator-node/test/tracks.js @@ -48,7 +48,7 @@ describe('test Tracks', function () { .post('/track_content') .attach('file', file, { filename: 'fname.jpg' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(400) }) @@ -74,7 +74,7 @@ describe('test Tracks', function () { .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(500) // Reset max file limits @@ -103,7 +103,7 @@ describe('test Tracks', function () { .post('/image_upload') .attach('file', file, { filename: 'fname.jpg' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(500) // Reset max file limits @@ -111,27 +111,27 @@ describe('test Tracks', function () { await server.close() }) - it('uploads file to IPFS', async function () { + it('uploads /track_content', async function () { const file = fs.readFileSync(testAudioFilePath) ipfsMock.addFromFs.exactly(33) ipfsMock.pin.add.exactly(33) - const resp1 = await request(app) + const trackContentResp = await request(app) .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) - assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') - assert.deepStrictEqual(resp1.body.track_segments.length, 32) - assert.deepStrictEqual(resp1.body.source_file.includes('.mp3'), true) - assert.deepStrictEqual(resp1.body.transcodedTrackCID, 'testCIDLink') - assert.deepStrictEqual(typeof resp1.body.transcodedTrackUUID, 'string') + assert.deepStrictEqual(trackContentResp.body.track_segments[0].multihash, 'testCIDLink') + assert.deepStrictEqual(trackContentResp.body.track_segments.length, 32) + assert.deepStrictEqual(trackContentResp.body.source_file.includes('.mp3'), true) + assert.deepStrictEqual(trackContentResp.body.transcodedTrackCID, 'testCIDLink') + assert.deepStrictEqual(typeof trackContentResp.body.transcodedTrackUUID, 'string') }) - // depends on "upload file to IPFS" + // depends on "uploads /track_content" it('creates Audius track', async function () { const file = fs.readFileSync(testAudioFilePath) @@ -139,34 +139,34 @@ describe('test Tracks', function () { ipfsMock.pin.add.exactly(34) libsMock.User.getUsers.exactly(2) - const resp1 = await request(app) + const trackContentResp = await request(app) .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) - assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') - assert.deepStrictEqual(resp1.body.track_segments.length, 32) - assert.deepStrictEqual(resp1.body.source_file.includes('.mp3'), true) + assert.deepStrictEqual(trackContentResp.body.track_segments[0].multihash, 'testCIDLink') + assert.deepStrictEqual(trackContentResp.body.track_segments.length, 32) + assert.deepStrictEqual(trackContentResp.body.source_file.includes('.mp3'), true) // creates Audius track const metadata = { test: 'field1', owner_id: 1, - track_segments: [{ 'multihash': 'testCIDLink', 'duration': 1000 }] + track_segments: trackContentResp.body.track_segments } - const resp2 = await request(app) + const trackMetadataResp = await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) - .send({ metadata, sourceFile: resp1.body.source_file }) + .set('X-Session-ID', session.sessionToken) + .send({ metadata, sourceFile: trackContentResp.body.source_file }) .expect(200) - assert.deepStrictEqual(resp2.body.metadataMultihash, 'testCIDLink') + assert.deepStrictEqual(trackMetadataResp.body.metadataMultihash, 'testCIDLink') }) - // depends on "upload file to IPFS" + // depends on "uploads /track_content" it('fails to create Audius track when segments not provided', async function () { const file = fs.readFileSync(testAudioFilePath) @@ -178,7 +178,7 @@ describe('test Tracks', function () { .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') @@ -193,12 +193,12 @@ describe('test Tracks', function () { await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata, sourceFile: resp1.body.source_file }) .expect(400) }) - // depends on "upload file to IPFS" + // depends on "uploads /track_content" it('fails to create Audius track when invalid segment multihashes are provided', async function () { const file = fs.readFileSync(testAudioFilePath) @@ -210,7 +210,7 @@ describe('test Tracks', function () { .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') @@ -226,12 +226,12 @@ describe('test Tracks', function () { await request(app) .post('/tracks') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata, sourceFile: resp1.body.source_file }) .expect(400) }) - // depends on "upload file to IPFS" + // depends on "uploads /track_content" it('fails to create Audius track when owner_id is not provided', async function () { const file = fs.readFileSync(testAudioFilePath) @@ -243,7 +243,7 @@ describe('test Tracks', function () { .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') @@ -258,12 +258,12 @@ describe('test Tracks', function () { await request(app) .post('/tracks') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata, sourceFile: resp1.body.source_file }) .expect(400) }) - // depends on "upload file to IPFS" and "creates Audius user" tests + // depends on "uploads /track_content" and "creates Audius track" tests it('completes Audius track creation', async function () { const file = fs.readFileSync(testAudioFilePath) @@ -271,41 +271,41 @@ describe('test Tracks', function () { ipfsMock.pin.add.exactly(34) libsMock.User.getUsers.exactly(4) - const resp1 = await request(app) + const trackContentResp = await request(app) .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) - assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') - assert.deepStrictEqual(resp1.body.track_segments.length, 32) - assert.deepStrictEqual(resp1.body.source_file.includes('.mp3'), true) + assert.deepStrictEqual(trackContentResp.body.track_segments[0].multihash, 'testCIDLink') + assert.deepStrictEqual(trackContentResp.body.track_segments.length, 32) + assert.deepStrictEqual(trackContentResp.body.source_file.includes('.mp3'), true) const metadata = { test: 'field1', - track_segments: [{ 'multihash': 'testCIDLink', 'duration': 1000 }], + track_segments: trackContentResp.body.track_segments, owner_id: 1 } - const resp2 = await request(app) + const trackMetadataResp = await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) - .send({ metadata, sourceFile: resp1.body.source_file }) + .set('X-Session-ID', session.sessionToken) + .send({ metadata, sourceFile: trackContentResp.body.source_file }) .expect(200) - if (resp2.body.metadataMultihash !== 'testCIDLink') { + if (trackMetadataResp.body.metadataMultihash !== 'testCIDLink') { throw new Error('invalid return data') } await request(app) .post('/tracks') - .set('X-Session-ID', session) - .send({ blockchainTrackId: 1, blockNumber: 10, metadataFileUUID: resp2.body.metadataFileUUID }) + .set('X-Session-ID', session.sessionToken) + .send({ blockchainTrackId: 1, blockNumber: 10, metadataFileUUID: trackMetadataResp.body.metadataFileUUID }) .expect(200) }) - // depends on "upload file to IPFS" + // depends on "uploads /track_content" it('fails to create downloadable track with no track_id and no source_id present', async function () { const file = fs.readFileSync(testAudioFilePath) @@ -317,7 +317,7 @@ describe('test Tracks', function () { .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') @@ -336,12 +336,12 @@ describe('test Tracks', function () { await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata }) .expect(400) }) - // depends on "upload file to IPFS" and "creates Audius user" tests + // depends on "uploads /track_content" and "creates Audius track" tests it('creates a downloadable track', async function () { const file = fs.readFileSync(testAudioFilePath) @@ -349,21 +349,21 @@ describe('test Tracks', function () { ipfsMock.pin.add.exactly(34) libsMock.User.getUsers.exactly(4) - const resp1 = await request(app) + const trackContentResp = await request(app) .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) - assert.deepStrictEqual(resp1.body.track_segments[0].multihash, 'testCIDLink') - assert.deepStrictEqual(resp1.body.track_segments.length, 32) - assert.deepStrictEqual(resp1.body.source_file.includes('.mp3'), true) + assert.deepStrictEqual(trackContentResp.body.track_segments[0].multihash, 'testCIDLink') + assert.deepStrictEqual(trackContentResp.body.track_segments.length, 32) + assert.deepStrictEqual(trackContentResp.body.source_file.includes('.mp3'), true) // needs debugging as to why this 'cid' key is needed for test to work const metadata = { test: 'field1', - track_segments: [{ 'multihash': 'testCIDLink', 'duration': 1000 }], + track_segments: trackContentResp.body.track_segments, owner_id: 1, download: { 'is_downloadable': true, @@ -372,20 +372,20 @@ describe('test Tracks', function () { } } - const resp2 = await request(app) + const trackMetadataResp = await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) - .send({ metadata, sourceFile: resp1.body.source_file }) + .set('X-Session-ID', session.sessionToken) + .send({ metadata, sourceFile: trackContentResp.body.source_file }) .expect(200) - if (resp2.body.metadataMultihash !== 'testCIDLink') { + if (trackMetadataResp.body.metadataMultihash !== 'testCIDLink') { throw new Error('invalid return data') } await request(app) .post('/tracks') - .set('X-Session-ID', session) - .send({ blockchainTrackId: 1, blockNumber: 10, metadataFileUUID: resp2.body.metadataFileUUID }) + .set('X-Session-ID', session.sessionToken) + .send({ blockchainTrackId: 1, blockNumber: 10, metadataFileUUID: trackMetadataResp.body.metadataFileUUID }) .expect(200) }) }) @@ -430,7 +430,7 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(500) }) @@ -442,7 +442,7 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(500) }) @@ -454,7 +454,7 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func .post('/track_content') .attach('file', file, { filename: 'fname.mp3' }) .set('Content-Type', 'multipart/form-data') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .expect(200) let storagePath = config.get('storagePath') @@ -491,7 +491,7 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func it('should throw an error if no metadata is passed', async function () { const resp = await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({}) .expect(400) @@ -508,7 +508,7 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func const resp = await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata }) .expect(403) @@ -525,11 +525,11 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func const resp = await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata }) .expect(500) - assert.deepStrictEqual(resp.body.error, 'Could not save file to disk, ipfs, and/or db: Error: ipfs add failed!') + assert.deepStrictEqual(resp.body.error, '/tracks/metadata saveFileFromBufferToIPFSAndDisk op failed: Error: ipfs add failed!') }) it('successfully adds metadata file to filesystem, db, and ipfs', async function () { @@ -541,7 +541,7 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func const resp = await request(app) .post('/tracks/metadata') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({ metadata }) .expect(function (res) { if (res.body.error) { diff --git a/creator-node/test/users.js b/creator-node/test/users.js index 7d0f233613b..27edd4ca34a 100644 --- a/creator-node/test/users.js +++ b/creator-node/test/users.js @@ -216,12 +216,12 @@ describe('test Users', function () { const session = await createStarterCNodeUser() await request(app) .post('/users/logout') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({}) .expect(200) await request(app) .post('/users/logout') - .set('X-Session-ID', session) + .set('X-Session-ID', session.sessionToken) .send({}) .expect(401) }) From 6f723c438a6f7a8969658740238f28be6e62cd11 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 17 Sep 2020 20:27:37 +0000 Subject: [PATCH 14/53] UNTESTED update sync with new ClockRecords --- creator-node/src/routes/nodeSync.js | 105 +++++++++++++++++----------- 1 file changed, 63 insertions(+), 42 deletions(-) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 68c1095f00e..b1b9613b4be 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -26,19 +26,20 @@ module.exports = function (app) { app.get('/export', handleResponse(async (req, res) => { const walletPublicKeys = req.query.wallet_public_key // array - const t = await models.sequelize.transaction() + const transaction = await models.sequelize.transaction() try { // Fetch cnodeUser for each walletPublicKey. - const cnodeUsers = await models.CNodeUser.findAll({ where: { walletPublicKey: walletPublicKeys }, transaction: t }) + const cnodeUsers = await models.CNodeUser.findAll({ where: { walletPublicKey: walletPublicKeys }, transaction }) const cnodeUserUUIDs = cnodeUsers.map((cnodeUser) => cnodeUser.cnodeUserUUID) - // Fetch all data for cnodeUserUUIDs: audiusUsers, tracks, files. - const [audiusUsers, tracks, files] = await Promise.all([ - models.AudiusUser.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction: t }), - models.Track.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction: t }), - models.File.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction: t }) + // Fetch all data for cnodeUserUUIDs: audiusUsers, tracks, files, clockRecords. + const [audiusUsers, tracks, files, clockRecords] = await Promise.all([ + models.AudiusUser.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }), + models.Track.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }), + models.File.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }), + models.ClockRecord.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }) ]) - await t.commit() + await transaction.commit() /** Bundle all data into cnodeUser objects to maximize import speed. */ @@ -51,27 +52,29 @@ module.exports = function (app) { cnodeUserDictObj['audiusUsers'] = [] cnodeUserDictObj['tracks'] = [] cnodeUserDictObj['files'] = [] + cnodeUserDictObj['clockRecords'] = [] cnodeUsersDict[cnodeUser.cnodeUserUUID] = cnodeUserDictObj }) audiusUsers.forEach(audiusUser => { - const audiusUserDictObj = audiusUser.toJSON() - cnodeUsersDict[audiusUserDictObj['cnodeUserUUID']]['audiusUsers'].push(audiusUserDictObj) + cnodeUsersDict[audiusUserDictObj['cnodeUserUUID']]['audiusUsers'].push(audiusUser.toJSON()) }) tracks.forEach(track => { - let trackDictObj = track.toJSON() - cnodeUsersDict[trackDictObj['cnodeUserUUID']]['tracks'].push(trackDictObj) + cnodeUsersDict[trackDictObj['cnodeUserUUID']]['tracks'].push(track.toJSON()) }) files.forEach(file => { - let fileDictObj = file.toJSON() - cnodeUsersDict[fileDictObj['cnodeUserUUID']]['files'].push(fileDictObj) + cnodeUsersDict[fileDictObj['cnodeUserUUID']]['files'].push(file.toJSON()) + }) + clockRecords.forEach(clockRecord => { + cnodeUsersDict[clockRecordDictObj['cnodeUserUUID']]['clockRecords'].push(clockRecord.toJSON()) }) // Expose ipfs node's peer ID. const ipfs = req.app.get('ipfsAPI') - let ipfsIDObj = await getIPFSPeerId(ipfs, config) + const ipfsIDObj = await getIPFSPeerId(ipfs, config) + // Rehydrate files if necessary for (let i = 0; i < files.length; i += RehydrateIPFSConcurrencyLimit) { const exportFilesSlice = files.slice(i, i + RehydrateIPFSConcurrencyLimit) req.logger.info(`Export rehydrateIpfs processing files ${i} to ${i + RehydrateIPFSConcurrencyLimit}`) @@ -92,7 +95,7 @@ module.exports = function (app) { } })) } - return successResponse({ cnodeUsers: cnodeUsersDict, ipfsIDObj: ipfsIDObj }) + return successResponse({ cnodeUsers: cnodeUsersDict, ipfsIDObj }) } catch (e) { await t.rollback() return errorResponseServerError(e.message) @@ -139,9 +142,10 @@ module.exports = function (app) { // Get & return latestBlockNumber for wallet const cnodeUser = await models.CNodeUser.findOne({ where: { walletPublicKey } }) - const latestBlockNumber = cnodeUser ? cnodeUser.latestBlockNumber : -1 + const latestBlockNumber = (cnodeUser) ? cnodeUser.latestBlockNumber : -1 + const clockValue = (cnodeUser) ? cnodeUser.clock : -1 - return successResponse({ walletPublicKey, latestBlockNumber }) + return successResponse({ walletPublicKey, latestBlockNumber, clockValue }) })) } @@ -217,18 +221,19 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { } const fetchedCnodeUserUUID = fetchedCNodeUser.cnodeUserUUID - const t = await models.sequelize.transaction() + const transaction = await models.sequelize.transaction() try { const cnodeUser = await models.CNodeUser.findOne({ where: { walletPublicKey: fetchedWalletPublicKey }, - transaction: t + transaction }) const fetchedLatestBlockNumber = fetchedCNodeUser.latestBlockNumber // Delete any previously stored data for cnodeUser in reverse table dependency order (cannot be parallelized). if (cnodeUser) { // Ensure imported data has higher blocknumber than already stored. + // TODO - replace this check with a clock check (!!!) const latestBlockNumber = cnodeUser.latestBlockNumber if ((fetchedLatestBlockNumber === -1 && latestBlockNumber !== -1) || (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber <= latestBlockNumber) @@ -241,35 +246,42 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { req.logger.info(redisKey, `beginning delete ops for cnodeUserUUID ${cnodeUserUUID}`) const numAudiusUsersDeleted = await models.AudiusUser.destroy({ - where: { cnodeUserUUID: cnodeUserUUID }, - transaction: t + where: { cnodeUserUUID }, + transaction }) req.logger.info(redisKey, `numAudiusUsersDeleted ${numAudiusUsersDeleted}`) + // TrackFiles must be deleted before associated Tracks can be deleted. const numTrackFilesDeleted = await models.File.destroy({ where: { - cnodeUserUUID: cnodeUserUUID, + cnodeUserUUID, trackUUID: { [models.Sequelize.Op.ne]: null } // Op.ne = notequal }, - transaction: t + transaction }) req.logger.info(redisKey, `numTrackFilesDeleted ${numTrackFilesDeleted}`) + const numTracksDeleted = await models.Track.destroy({ - where: { cnodeUserUUID: cnodeUserUUID }, - transaction: t + where: { cnodeUserUUID }, + transaction }) req.logger.info(redisKey, `numTracksDeleted ${numTracksDeleted}`) + // Delete all remaining files (image / metadata files). const numNonTrackFilesDeleted = await models.File.destroy({ - where: { cnodeUserUUID: cnodeUserUUID }, - transaction: t + where: { cnodeUserUUID }, + transaction }) req.logger.info(redisKey, `numNonTrackFilesDeleted ${numNonTrackFilesDeleted}`) + + const numClockRecordsDeleted = await models.ClockRecord.destroy({ + where: { cnodeUserUUID }, + transaction + }) + req.logger.info(redisKey, `numClockRecordsDeleted ${numClockRecordsDeleted}`) // Delete cnodeUser entry - await cnodeUser.destroy({ - transaction: t - }) + await cnodeUser.destroy({transaction}) req.logger.info(redisKey, `deleted cnodeUserEntry`) } @@ -284,13 +296,23 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { latestBlockNumber: fetchedLatestBlockNumber, lastLogin: fetchedCNodeUser.lastLogin, clock: fetchedCNodeUser.clock - }, { transaction: t }) + }, { transaction }) req.logger.info(redisKey, `Inserted nodeUser for cnodeUserUUID ${fetchedCnodeUserUUID}`) - // Make list of all track Files to add after track creation. + // Save all clockRecords to DB + await models.ClockRecord.bulkCreate(fetchedCNodeUser.clockRecords.map(clockRecord => ({ + ...clockRecord, + cnodeUserUUID: fetchedCnodeUserUUID + })), { transaction }) + req.logger.info(redisKey, 'Recorded all ClockRecord entries in DB') + + /* + * Make list of all track Files to add after track creation + * + * Files with trackUUIDs cannot be created until tracks have been created, + * but tracks cannot be created until metadata and cover art files have been created. + */ - // Files with trackUUIDs cannot be created until tracks have been created, - // but tracks cannot be created until metadata and cover art files have been created. const trackFiles = fetchedCNodeUser.files.filter(file => models.File.TrackTypes.includes(file.type)) const nonTrackFiles = fetchedCNodeUser.files.filter(file => models.File.NonTrackTypes.includes(file.type)) @@ -302,8 +324,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { trackFile => saveFileForMultihash(req, trackFile.multihash, trackFile.storagePath, userReplicaSet) )) } - - req.logger.info('Saved all track files to disk.') + req.logger.info(redisKey, 'Saved all track files to disk.') // Save all non-track files to disk in batches (to limit concurrent load) for (let i = 0; i < nonTrackFiles.length; i += NonTrackFileSaveConcurrencyLimit) { @@ -325,7 +346,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { } )) } - req.logger.info('Saved all non-track files to disk.') + req.logger.info(redisKey, 'Saved all non-track files to disk.') await models.File.bulkCreate(nonTrackFiles.map(file => ({ fileUUID: file.fileUUID, @@ -338,7 +359,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { fileName: file.fileName, dirMultihash: file.dirMultihash, clock: file.clock - })), { transaction: t }) + })), { transaction }) req.logger.info(redisKey, 'created all non-track files') await models.Track.bulkCreate(fetchedCNodeUser.tracks.map(track => ({ @@ -349,7 +370,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { metadataFileUUID: track.metadataFileUUID, coverArtFileUUID: track.coverArtFileUUID, clock: track.clock - })), { transaction: t }) + })), { transaction }) req.logger.info(redisKey, 'created all tracks') // Save all track files to db @@ -364,7 +385,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { fileName: trackFile.fileName, dirMultihash: trackFile.dirMultihash, clock: trackFile.clock - })), { transaction: t }) + })), { transaction }) req.logger.info('saved all track files to db') await models.AudiusUser.bulkCreate(fetchedCNodeUser.audiusUsers.map(audiusUser => ({ @@ -376,7 +397,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { coverArtFileUUID: audiusUser.coverArtFileUUID, profilePicFileUUID: audiusUser.profilePicFileUUID, clock: audiusUser.clock - })), { transaction: t }) + })), { transaction }) req.logger.info('saved all audiususer data to db') await t.commit() From ec4aa1fdb89d42c9291ad92539f081167835c19b Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Fri, 18 Sep 2020 10:44:32 -0400 Subject: [PATCH 15/53] Append-only schema and migration changes for clockwerk (#827) * Migration to add trackBlockchainId to Files and remove extraneous UUID colums * lint * Start modifying routes to work with blockchainId for tracks * bug fixes * simplify nodesync create db props * Add read only mode * Lint fix * excellent test coverage * migration changes * More fixes * db only nodesync * code review fixes * bug fixes --- ...845-allow-track-and-audiusUsers-appends.js | 82 ++++++++++++ creator-node/src/app.js | 2 + creator-node/src/config.js | 6 + .../readOnly/readOnlyMiddleware.js | 28 +++++ .../readOnly/readOnlyMiddleware.test.js | 61 +++++++++ creator-node/src/models/audiususer.js | 9 +- creator-node/src/models/file.js | 12 +- creator-node/src/models/track.js | 14 +-- creator-node/src/routes/audiusUsers.js | 2 +- creator-node/src/routes/nodeSync.js | 117 +++++++----------- creator-node/src/routes/tracks.js | 76 +++++++----- creator-node/test/lib/reqMock.js | 44 +++++++ 12 files changed, 321 insertions(+), 132 deletions(-) create mode 100644 creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js create mode 100644 creator-node/src/middlewares/readOnly/readOnlyMiddleware.js create mode 100644 creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js create mode 100644 creator-node/test/lib/reqMock.js diff --git a/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js new file mode 100644 index 00000000000..a1c4ad457d3 --- /dev/null +++ b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js @@ -0,0 +1,82 @@ +'use strict' + +module.exports = { + up: async (queryInterface, Sequelize) => { + // Scope of the migration is: + // Remove trackUUID from Files and replace it with trackBlockchainId and migrate all existing values over + // Remove trackUUID field from Tracks table + // Remove UNIQUE constraint for blockchainId, trackUUID in Tracks table + // Add NOT NULL constraint for blockchainId in Tracks table + // Remove audiusUserUUID field from AudiusUsers table + // Remove UNIQUE constraint for audiusUserUUID in AudiusUsers table (no unique constraint on blockchainId) + // Add NOT NULL constraint for blockchainId in AudiusUsers table + + await queryInterface.sequelize.query(` + BEGIN; + -- replace Files table in place with extra trackBlockchainId column and drops the trackUUID column + CREATE TABLE "Files_new" ( like "Files" ); + ALTER TABLE "Files_new" ADD COLUMN "trackBlockchainId" INTEGER; + INSERT INTO "Files_new" ("multihash", "sourceFile", "storagePath", "createdAt", "updatedAt", "fileUUID", "cnodeUserUUID", "trackUUID", "type", "fileName", "dirMultihash", "trackBlockchainId") SELECT f."multihash", f."sourceFile", f."storagePath", f."createdAt", f."updatedAt", f."fileUUID", f."cnodeUserUUID", f."trackUUID", f."type", f."fileName", f."dirMultihash", t."blockchainId" FROM "Files" f LEFT OUTER JOIN "Tracks" t ON f."trackUUID" = t."trackUUID"; + ALTER TABLE "Files_new" DROP COLUMN "trackUUID"; + ALTER TABLE "Files" RENAME TO "Files_old"; + ALTER TABLE "Files_new" RENAME TO "Files"; + DROP TABLE "Files_old" CASCADE; + + -- add pkey + -- ALTER TABLE "Files" ADD CONSTRAINT "Files_fileUUID_key" UNIQUE ("fileUUID"); + ALTER TABLE "Files" ADD PRIMARY KEY ("fileUUID"); + + -- add back indexes + CREATE INDEX "Files_multihash_idx" ON public."Files" USING btree (multihash); + CREATE INDEX "Files_cnodeUserUUID_idx" ON public."Files" USING btree ("cnodeUserUUID"); + CREATE INDEX "Files_dir_multihash_idx" ON public."Files" USING btree ("dirMultihash"); + CREATE INDEX "Files_trackBlockchainId_idx" ON public."Files" USING btree ("trackBlockchainId"); + + -- add in the foreign key constraint from Files to other tables + -- No fkey from Files to Tracks because we don't have a unique constraint on trackUUID or blockchainId on Tracks so postgres would reject the fkey + ALTER TABLE "Files" ADD CONSTRAINT "Files_cnodeUserUUID_fkey" FOREIGN KEY ("cnodeUserUUID") REFERENCES "CNodeUsers" ("cnodeUserUUID") ON DELETE RESTRICT; + + -- 5 foreign key constraints get dropped in the CASCADE, so add them back in for the new table + ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_coverArtFileUUID_fkey" FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_profilePicFileUUID_fkey" FOREIGN KEY ("profilePicFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_coverArtFileUUID_fkey " FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + + -- remove the unique constraints from Tracks + ALTER TABLE "Tracks" DROP CONSTRAINT "Tracks_trackUUID_key"; + ALTER TABLE "Tracks" DROP CONSTRAINT "blockchainId_unique_idx"; + + -- add a not null constraint to Tracks blockchainId, just run a delete query to remove any outstanding Tracks without a blockchainId in case there are any + DELETE FROM "Tracks" WHERE "blockchainId" IS NULL; + ALTER TABLE "Tracks" ALTER COLUMN "blockchainId" SET NOT NULL; + + -- remove the trackUUID column from Tracks + ALTER TABLE "Tracks" DROP COLUMN "trackUUID"; + + -- remove the unique constraint from AudiusUsers + ALTER TABLE "AudiusUsers" DROP CONSTRAINT "AudiusUsers_audiusUserUUID_key"; + + -- add a not null constraint to AudiusUsers blockchainId, just run a delete query to remove any outstanding AudiusUsers without a blockchainId in case there are any + DELETE FROM "AudiusUsers" WHERE "blockchainId" IS NULL; + ALTER TABLE "AudiusUsers" ALTER COLUMN "blockchainId" SET NOT NULL; + + -- remove the audiusUserUUID field as the AudiusUsers pkey + ALTER TABLE "AudiusUsers" DROP CONSTRAINT "AudiusUsers_pkey"; + + -- remove the audiusUserUUID column from AudiusUsers + ALTER TABLE "AudiusUsers" DROP COLUMN "audiusUserUUID"; + + COMMIT; + `) + // TODO - add a primary key to Tracks (blockchainId:clock) + // TODO - add a primary key to AudiusUsers (blockchainId:clock) + // TODO - remove Files unique constraint since pkey does that + }, + + down: (queryInterface, Sequelize) => { + /* + The up migration destroys tons of information, if we need to revert the best option is to restore from a snapshot + */ + } +} diff --git a/creator-node/src/app.js b/creator-node/src/app.js index 73182fa5f1d..acca01499ba 100644 --- a/creator-node/src/app.js +++ b/creator-node/src/app.js @@ -5,6 +5,7 @@ const cors = require('cors') const { sendResponse, errorResponseServerError } = require('./apiHelpers') const { logger, loggingMiddleware } = require('./logging') const { userNodeMiddleware } = require('./userNodeMiddleware') +const { readOnlyMiddleware } = require('./middlewares/readOnly/readOnlyMiddleware') const { userReqLimiter, trackReqLimiter, audiusUserReqLimiter, metadataReqLimiter, imageReqLimiter } = require('./reqLimiter') const config = require('./config') const healthCheckRoutes = require('./components/healthCheck/healthCheckController') @@ -15,6 +16,7 @@ const app = express() app.use(loggingMiddleware) app.use(bodyParser.json({ limit: '1mb' })) app.use(userNodeMiddleware) +app.use(readOnlyMiddleware) app.use(cors()) // Rate limit routes diff --git a/creator-node/src/config.js b/creator-node/src/config.js index c403add28aa..a6f8f0e9d52 100644 --- a/creator-node/src/config.js +++ b/creator-node/src/config.js @@ -288,6 +288,12 @@ const config = convict({ env: 'isUserMetadataNode', default: false }, + isReadOnlyMode: { + doc: 'Flag indicating whether to run this node in read only mode (no writes)', + format: Boolean, + env: 'isReadOnlyMode', + default: false + }, userMetadataNodeUrl: { doc: 'address for user metadata node', format: String, diff --git a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js new file mode 100644 index 00000000000..6694f37552a --- /dev/null +++ b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js @@ -0,0 +1,28 @@ +const { sendResponse, errorResponseServerError } = require('../../apiHelpers') +const config = require('../../config') + +/** + * Middleware to block all non-GET api calls if the server should be in "read-only" mode + */ +function readOnlyMiddleware (req, res, next) { + const isReadOnlyMode = config.get('isReadOnlyMode') + const method = req.method + const canProceed = readOnlyMiddlewareHelper(isReadOnlyMode, method) + + if (!canProceed) return sendResponse(req, res, errorResponseServerError('Server is in read-only mode at the moment')) + next() +} + +/** + * + * @param {Boolean} isReadOnlyMode From config.get('isReadOnlyMode') + * @param {String} method REST method for this request eg. POST, GET + * @returns {Boolean} returns true if the request can proceed. eg GET in read only or any request in non read-only mode + */ +function readOnlyMiddlewareHelper (isReadOnlyMode, method) { + if (isReadOnlyMode && method !== 'GET') return false + + return true +} + +module.exports = { readOnlyMiddleware, readOnlyMiddlewareHelper } diff --git a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js new file mode 100644 index 00000000000..5b88dcab6da --- /dev/null +++ b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js @@ -0,0 +1,61 @@ +const { readOnlyMiddleware, readOnlyMiddlewareHelper } = require('./readOnlyMiddleware') +const assert = require('assert') +const config = require('../../config') +const { resFactory, loggerFactory } = require('../../../test/lib/reqMock') + +describe('Test read-only middleware', function () { + beforeEach(function() { + config.reset('isReadOnlyMode') + }) + + it('Should pass if read-only enabled and is GET request', function () { + const method = 'GET' + const isReadOnlyMode = true + config.set('isReadOnlyMode', isReadOnlyMode) + let nextCalled = false + + + readOnlyMiddleware({ method }, {}, function() { + nextCalled = true + }) + + assert.deepStrictEqual(readOnlyMiddlewareHelper(isReadOnlyMode, method), true) + assert.deepStrictEqual(nextCalled, true) + }) + + it('Should fail if read-only enabled and is not GET request', function () { + const isReadOnlyMode = true + const method = 'POST' + config.set('isReadOnlyMode', isReadOnlyMode) + let nextCalled = false + + const logger = loggerFactory() + const req = { + method, + logger + } + const res = resFactory() + + readOnlyMiddleware(req, res, function() { + nextCalled = true + }) + + assert.deepStrictEqual(res.statusCode, 500) + assert.deepStrictEqual(readOnlyMiddlewareHelper(isReadOnlyMode, method), false) + assert.deepStrictEqual(nextCalled, false) + }) + + it('Should pass if read-only not enabled and is POST request', function () { + const method = 'POST' + const isReadOnlyMode = false + config.set('isReadOnlyMode', isReadOnlyMode) + let nextCalled = false + + readOnlyMiddleware({ method }, {}, function() { + nextCalled = true + }) + + assert.deepStrictEqual(readOnlyMiddlewareHelper(isReadOnlyMode, method), true) + assert.deepStrictEqual(nextCalled, true) + }) +}) diff --git a/creator-node/src/models/audiususer.js b/creator-node/src/models/audiususer.js index 409d255729e..77cbdde8cd1 100644 --- a/creator-node/src/models/audiususer.js +++ b/creator-node/src/models/audiususer.js @@ -1,16 +1,9 @@ 'use strict' module.exports = (sequelize, DataTypes) => { const AudiusUser = sequelize.define('AudiusUser', { - audiusUserUUID: { - type: DataTypes.UUID, - allowNull: false, - primaryKey: true, - defaultValue: DataTypes.UUIDV4 - }, blockchainId: { type: DataTypes.BIGINT, - unique: true, - allowNull: true + allowNull: false }, cnodeUserUUID: { type: DataTypes.UUID, diff --git a/creator-node/src/models/file.js b/creator-node/src/models/file.js index bcc58a3638b..6d484b0ee78 100644 --- a/creator-node/src/models/file.js +++ b/creator-node/src/models/file.js @@ -13,8 +13,8 @@ module.exports = (sequelize, DataTypes) => { allowNull: false }, // only non-null for track files (as opposed to image/metadata files) - trackUUID: { - type: DataTypes.UUID, + trackBlockchainId: { + type: DataTypes.INTEGER, allowNull: true // `true` as we use File entries for more than just uploaded tracks }, multihash: { @@ -67,6 +67,9 @@ module.exports = (sequelize, DataTypes) => { }, { fields: ['dirMultihash'] + }, + { + fields: ['trackBlockchainId'] } ] }) @@ -77,11 +80,6 @@ module.exports = (sequelize, DataTypes) => { sourceKey: 'cnodeUserUUID', onDelete: 'RESTRICT' }) - File.belongsTo(models.Track, { - foreignKey: 'trackUUID', - sourceKey: 'trackUUID', - onDelete: 'RESTRICT' - }) } // TODO why no work? diff --git a/creator-node/src/models/track.js b/creator-node/src/models/track.js index c9f2278f496..35539dcecbc 100644 --- a/creator-node/src/models/track.js +++ b/creator-node/src/models/track.js @@ -2,12 +2,6 @@ module.exports = (sequelize, DataTypes) => { const Track = sequelize.define('Track', { - trackUUID: { - type: DataTypes.UUID, - allowNull: false, - primaryKey: true, - defaultValue: DataTypes.UUIDV4 - }, cnodeUserUUID: { type: DataTypes.UUID, allowNull: false @@ -22,8 +16,7 @@ module.exports = (sequelize, DataTypes) => { }, blockchainId: { type: DataTypes.BIGINT, - allowNull: true, - unique: true + allowNull: false, }, coverArtFileUUID: { type: DataTypes.UUID, @@ -44,11 +37,6 @@ module.exports = (sequelize, DataTypes) => { targetKey: 'cnodeUserUUID', onDelete: 'RESTRICT' }) - Track.belongsTo(models.File, { // belongsTo, or hasMany? - foreignKey: 'trackUUID', - targetKey: 'trackUUID', - onDelete: 'RESTRICT' - }) Track.belongsTo(models.File, { // belongsTo, or hasOne foreignKey: 'metadataFileUUID', targetKey: 'fileUUID', diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index 74a0b040d5b..dd32f41f5bd 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -120,7 +120,7 @@ module.exports = function (app) { await transaction.commit() triggerSecondarySyncs(req) - return successResponse({ audiusUserUUID: audiusUser.audiusUserUUID }) + return successResponse() } catch (e) { await transaction.rollback() return errorResponseServerError(e.message) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index b1b9613b4be..6ba1f8affab 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -97,7 +97,7 @@ module.exports = function (app) { } return successResponse({ cnodeUsers: cnodeUsersDict, ipfsIDObj }) } catch (e) { - await t.rollback() + await transaction.rollback() return errorResponseServerError(e.message) } })) @@ -110,6 +110,8 @@ module.exports = function (app) { const walletPublicKeys = req.body.wallet // array const creatorNodeEndpoint = req.body.creator_node_endpoint // string const immediate = (req.body.immediate === true || req.body.immediate === 'true') + // option to sync just the db records as opposed to db records and files on disk, defaults to false + const dbOnlySync = (req.body.db_only_sync === true || req.body.db_only_sync === 'true') if (!immediate) { req.logger.info('debounce time', config.get('debounceTime')) @@ -120,13 +122,13 @@ module.exports = function (app) { req.logger.info('clear timeout for', wallet, 'time', Date.now()) } syncQueue[wallet] = setTimeout( - async () => _nodesync(req, [wallet], creatorNodeEndpoint), + async () => _nodesync(req, [wallet], creatorNodeEndpoint, dbOnlySync), config.get('debounceTime') ) req.logger.info('set timeout for', wallet, 'time', Date.now()) } } else { - await _nodesync(req, walletPublicKeys, creatorNodeEndpoint) + await _nodesync(req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync) } return successResponse() })) @@ -149,7 +151,7 @@ module.exports = function (app) { })) } -async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { +async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync) { const start = Date.now() req.logger.info('begin nodesync for ', walletPublicKeys, 'time', start) @@ -255,7 +257,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { const numTrackFilesDeleted = await models.File.destroy({ where: { cnodeUserUUID, - trackUUID: { [models.Sequelize.Op.ne]: null } // Op.ne = notequal + trackBlockchainId: { [models.Sequelize.Op.ne]: null } // Op.ne = notequal }, transaction }) @@ -309,94 +311,71 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint) { /* * Make list of all track Files to add after track creation * - * Files with trackUUIDs cannot be created until tracks have been created, + * Files with trackBlockchainIds cannot be created until tracks have been created, * but tracks cannot be created until metadata and cover art files have been created. */ const trackFiles = fetchedCNodeUser.files.filter(file => models.File.TrackTypes.includes(file.type)) const nonTrackFiles = fetchedCNodeUser.files.filter(file => models.File.NonTrackTypes.includes(file.type)) - // Save all track files to disk in batches (to limit concurrent load) - for (let i = 0; i < trackFiles.length; i += TrackSaveConcurrencyLimit) { - const trackFilesSlice = trackFiles.slice(i, i + TrackSaveConcurrencyLimit) - req.logger.info(`TrackFiles saveFileForMultihash - processing trackFiles ${i} to ${i + TrackSaveConcurrencyLimit}...`) - await Promise.all(trackFilesSlice.map( - trackFile => saveFileForMultihash(req, trackFile.multihash, trackFile.storagePath, userReplicaSet) - )) - } - req.logger.info(redisKey, 'Saved all track files to disk.') - - // Save all non-track files to disk in batches (to limit concurrent load) - for (let i = 0; i < nonTrackFiles.length; i += NonTrackFileSaveConcurrencyLimit) { - const nonTrackFilesSlice = nonTrackFiles.slice(i, i + NonTrackFileSaveConcurrencyLimit) - req.logger.info(`NonTrackFiles saveFileForMultihash - processing files ${i} to ${i + NonTrackFileSaveConcurrencyLimit}...`) - await Promise.all(nonTrackFilesSlice.map( - nonTrackFile => { - // Skip over directories since there's no actual content to sync - // The files inside the directory are synced separately - if (nonTrackFile.type !== 'dir') { - // if it's an image file, we need to pass in the actual filename because the gateway request is /ipfs/Qm123/ - // need to also check fileName is not null to make sure it's a dir-style image. non-dir images won't have a 'fileName' db column - if (nonTrackFile.type === 'image' && nonTrackFile.fileName !== null) { - return saveFileForMultihash(req, nonTrackFile.multihash, nonTrackFile.storagePath, userReplicaSet, nonTrackFile.fileName) - } else { - return saveFileForMultihash(req, nonTrackFile.multihash, nonTrackFile.storagePath, userReplicaSet) + // if not just db records sync, sync everything + if (!dbOnlySync) { + // Save all track files to disk in batches (to limit concurrent load) + for (let i = 0; i < trackFiles.length; i += TrackSaveConcurrencyLimit) { + const trackFilesSlice = trackFiles.slice(i, i + TrackSaveConcurrencyLimit) + req.logger.info(`TrackFiles saveFileForMultihash - processing trackFiles ${i} to ${i + TrackSaveConcurrencyLimit}...`) + await Promise.all(trackFilesSlice.map( + trackFile => saveFileForMultihash(req, trackFile.multihash, trackFile.storagePath, userReplicaSet) + )) + } + req.logger.info(redisKey, 'Saved all track files to disk.') + + // Save all non-track files to disk in batches (to limit concurrent load) + for (let i = 0; i < nonTrackFiles.length; i += NonTrackFileSaveConcurrencyLimit) { + const nonTrackFilesSlice = nonTrackFiles.slice(i, i + NonTrackFileSaveConcurrencyLimit) + req.logger.info(`NonTrackFiles saveFileForMultihash - processing files ${i} to ${i + NonTrackFileSaveConcurrencyLimit}...`) + await Promise.all(nonTrackFilesSlice.map( + nonTrackFile => { + // Skip over directories since there's no actual content to sync + // The files inside the directory are synced separately + if (nonTrackFile.type !== 'dir') { + // if it's an image file, we need to pass in the actual filename because the gateway request is /ipfs/Qm123/ + // need to also check fileName is not null to make sure it's a dir-style image. non-dir images won't have a 'fileName' db column + if (nonTrackFile.type === 'image' && nonTrackFile.fileName !== null) { + return saveFileForMultihash(req, nonTrackFile.multihash, nonTrackFile.storagePath, userReplicaSet, nonTrackFile.fileName) + } else { + return saveFileForMultihash(req, nonTrackFile.multihash, nonTrackFile.storagePath, userReplicaSet) + } } } - } - )) + )) + } + req.logger.info('Saved all non-track files to disk.') } - req.logger.info(redisKey, 'Saved all non-track files to disk.') await models.File.bulkCreate(nonTrackFiles.map(file => ({ - fileUUID: file.fileUUID, - trackUUID: null, - cnodeUserUUID: fetchedCnodeUserUUID, - multihash: file.multihash, - sourceFile: file.sourceFile, - storagePath: file.storagePath, - type: file.type, - fileName: file.fileName, - dirMultihash: file.dirMultihash, - clock: file.clock + ...file, + trackBlockchainId: null, + cnodeUserUUID: fetchedCnodeUserUUID })), { transaction }) req.logger.info(redisKey, 'created all non-track files') await models.Track.bulkCreate(fetchedCNodeUser.tracks.map(track => ({ - trackUUID: track.trackUUID, - blockchainId: track.blockchainId, - cnodeUserUUID: fetchedCnodeUserUUID, - metadataJSON: track.metadataJSON, - metadataFileUUID: track.metadataFileUUID, - coverArtFileUUID: track.coverArtFileUUID, - clock: track.clock + ...track, + cnodeUserUUID: fetchedCnodeUserUUID })), { transaction }) req.logger.info(redisKey, 'created all tracks') // Save all track files to db await models.File.bulkCreate(trackFiles.map(trackFile => ({ - fileUUID: trackFile.fileUUID, - trackUUID: trackFile.trackUUID, - cnodeUserUUID: fetchedCnodeUserUUID, - multihash: trackFile.multihash, - sourceFile: trackFile.sourceFile, - storagePath: trackFile.storagePath, - type: trackFile.type, - fileName: trackFile.fileName, - dirMultihash: trackFile.dirMultihash, - clock: trackFile.clock + ...trackFile, + cnodeUserUUID: fetchedCnodeUserUUID })), { transaction }) req.logger.info('saved all track files to db') await models.AudiusUser.bulkCreate(fetchedCNodeUser.audiusUsers.map(audiusUser => ({ - audiusUserUUID: audiusUser.audiusUserUUID, - cnodeUserUUID: fetchedCnodeUserUUID, - blockchainId: audiusUser.blockchainId, - metadataJSON: audiusUser.metadataJSON, - metadataFileUUID: audiusUser.metadataFileUUID, - coverArtFileUUID: audiusUser.coverArtFileUUID, - profilePicFileUUID: audiusUser.profilePicFileUUID, - clock: audiusUser.clock + ...audiusUser, + cnodeUserUUID: fetchedCnodeUserUUID })), { transaction }) req.logger.info('saved all audiususer data to db') diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 10fd2fe5c40..60e7aea4a26 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -205,11 +205,12 @@ module.exports = function (app) { // See if the track already has a transcoded master if (trackId) { - const { trackUUID } = await models.Track.findOne({ - attributes: ['trackUUID'], + const { blockchainId } = await models.Track.findOne({ + attributes: ['blockchainId'], where: { blockchainId: trackId - } + }, + order: [['clock', 'DESC']] }) // Error if no DB entry for transcode found @@ -218,7 +219,7 @@ module.exports = function (app) { where: { cnodeUserUUID: req.session.cnodeUserUUID, type: 'copy320', - trackUUID + trackBlockchainId: blockchainId } }) if (!transcodedFile) { @@ -322,10 +323,11 @@ module.exports = function (app) { const existingTrackEntry = await models.Track.findOne({ where: { cnodeUserUUID, - metadataFileUUID, + // metadataFileUUID, blockchainId: blockchainTrackId, coverArtFileUUID }, + order: [['clock', 'DESC']], transaction }) @@ -351,7 +353,7 @@ module.exports = function (app) { const trackSegmentCIDs = metadataJSON.track_segments.map(segment => segment.multihash) - // if track created, ensure files exist with trackuuid = null and update them + // if track created, ensure files exist with trackBlockchainId = null and update them if (!existingTrackEntry) { // Associate the transcode file db record with trackUUID if (transcodedTrackUUID) { @@ -359,7 +361,7 @@ module.exports = function (app) { where: { fileUUID: transcodedTrackUUID, cnodeUserUUID, - trackUUID: null, + trackBlockchainId: null, type: 'copy320' }, transaction @@ -368,11 +370,11 @@ module.exports = function (app) { throw new Error('Did not find a transcoded file for the provided CID.') } const numAffectedRows = await models.File.update( - { trackUUID: track.trackUUID }, + { trackBlockchainId: track.blockchainId }, { where: { fileUUID: transcodedTrackUUID, cnodeUserUUID, - trackUUID: null, + trackBlockchainId: null, type: 'copy320' // TODO - replace with model enum }, transaction @@ -388,7 +390,7 @@ module.exports = function (app) { where: { multihash: trackSegmentCIDs, cnodeUserUUID, - trackUUID: null, + trackBlockchainId: null, type: 'track' }, transaction @@ -398,28 +400,29 @@ module.exports = function (app) { throw new Error('Did not find files for every track segment CID.') } const numAffectedRows = await models.File.update( - { trackUUID: track.trackUUID }, - { where: { - multihash: trackSegmentCIDs, - cnodeUserUUID, - trackUUID: null, - type: 'track' - }, - transaction + { trackBlockchainId: track.blockchainId }, + { + where: { + multihash: trackSegmentCIDs, + cnodeUserUUID, + trackBlockchainId: null, + type: 'track' + }, + transaction } ) - if (numAffectedRows !== trackSegmentCIDs.length) { logger.error(`\n\n\nnumAffectedRows: ${numAffectedRows}`) + if (parseInt(numAffectedRows, 10) !== trackSegmentCIDs.length) { throw new Error('Failed to associate files for every track segment CID.') } - } else { /** If track updated, ensure files exist with trackuuid. */ + } else { /** If track updated, ensure files exist with trackBlockchainId. */ // Ensure transcode file db record exists if uuid provided if (transcodedTrackUUID) { const transcodedFile = await models.File.findOne({ where: { fileUUID: transcodedTrackUUID, cnodeUserUUID, - trackUUID: track.trackUUID, + trackBlockchainId: track.blockchainId, type: 'copy320' }, transaction @@ -434,13 +437,13 @@ module.exports = function (app) { where: { multihash: trackSegmentCIDs, cnodeUserUUID, - trackUUID: track.trackUUID, + trackBlockchainId: track.blockchainId, type: 'track' }, transaction }) if (trackFiles.length < trackSegmentCIDs.length) { - throw new Error('Did not find files for every track segment CID with trackUUID.') + throw new Error('Did not find files for every track segment CID with trackBlockchainId.') } } @@ -464,7 +467,7 @@ module.exports = function (app) { logger.info(`completed POST tracks route`) await transaction.commit() triggerSecondarySyncs(req) - return successResponse({ trackUUID: track.trackUUID }) + return successResponse() } catch (e) { logger.error(e.message) await transaction.rollback() @@ -479,7 +482,10 @@ module.exports = function (app) { return errorResponseBadRequest('Please provide blockchainId.') } - const track = await models.Track.findOne({ where: { blockchainId } }) + const track = await models.Track.findOne({ + where: { blockchainId }, + order: [['clock', 'DESC']] + }) if (!track) { return errorResponseBadRequest(`No track found for blockchainId ${blockchainId}`) } @@ -490,11 +496,11 @@ module.exports = function (app) { } // Case: track is marked as downloadable - // - Check if downloadable file exists. Since copyFile may or may not have trackUUID association, - // fetch a segmentFile for trackUUID, and find copyFile for segmentFile's sourceFile. + // - Check if downloadable file exists. Since copyFile may or may not have trackBlockchainId association, + // fetch a segmentFile for trackBlockchainId, and find copyFile for segmentFile's sourceFile. const segmentFile = await models.File.findOne({ where: { type: 'track', - trackUUID: track.trackUUID + trackBlockchainId: track.blockchainId } }) const copyFile = await models.File.findOne({ where: { type: 'copy320', @@ -533,12 +539,13 @@ module.exports = function (app) { return errorResponseBadRequest(`Invalid ID: ${encodedId}`) } - const { trackUUID } = await models.Track.findOne({ - attributes: ['trackUUID'], - where: { blockchainId } + const { blockchainId: blockchainIdFromTrack } = await models.Track.findOne({ + attributes: ['blockchainId'], + where: { blockchainId }, + order: [['clock', 'DESC']] }) - if (!trackUUID) { + if (!blockchainIdFromTrack) { return errorResponseBadRequest(`No track found for blockchainId ${blockchainId}`) } @@ -546,8 +553,9 @@ module.exports = function (app) { attributes: ['multihash'], where: { type: 'copy320', - trackUUID - } + trackBlockchainId: blockchainIdFromTrack + }, + order: [['clock', 'DESC']] }) if (!multihash) { diff --git a/creator-node/test/lib/reqMock.js b/creator-node/test/lib/reqMock.js new file mode 100644 index 00000000000..26c66ddffc9 --- /dev/null +++ b/creator-node/test/lib/reqMock.js @@ -0,0 +1,44 @@ +/** + * Mocks for Express res object and bunyan logger for unit tests + */ + +/** + * Creates a new response object that can be called like `res.statusCode().send()`. Also preserves + * status code in a parameter that's accessible by the test + */ +const resFactory = () => { + return { + statusCode: null, + status: function (statusCode) { + this.statusCode = statusCode + return { + send: () => {} + } + }, + set: () => {} + } +} + +/** + * Logger object with support for info, warn, error and creating child loggers + */ +const logger = { + child: () => { + return { + ...logger + } + }, + info: () => {}, + warn: () => {}, + error: () => {} +} + +/** + * Creates a new logger object + */ +const loggerFactory = () => { + return logger +} + + +module.exports = { resFactory, loggerFactory } \ No newline at end of file From 067c2f99217e75aa0ceaf356345c9bbc9a6e8d9a Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 18 Sep 2020 15:09:13 +0000 Subject: [PATCH 16/53] Re-order clock migration --- ...20200819145320-vector-clock.js => 20200918150546-add-clock.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename creator-node/sequelize/migrations/{20200819145320-vector-clock.js => 20200918150546-add-clock.js} (100%) diff --git a/creator-node/sequelize/migrations/20200819145320-vector-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js similarity index 100% rename from creator-node/sequelize/migrations/20200819145320-vector-clock.js rename to creator-node/sequelize/migrations/20200918150546-add-clock.js From 8fa5083ba2eca4a0e6c650a9d75656ec723f659f Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 18 Sep 2020 15:27:28 +0000 Subject: [PATCH 17/53] Fix some things --- ...845-allow-track-and-audiusUsers-appends.js | 3 -- .../migrations/20200918150546-add-clock.js | 42 +++++-------------- .../readOnly/readOnlyMiddleware.js | 3 +- .../readOnly/readOnlyMiddleware.test.js | 9 ++-- creator-node/src/models/track.js | 2 +- creator-node/src/routes/audiusUsers.js | 2 +- creator-node/src/routes/nodeSync.js | 26 +++++++----- creator-node/src/routes/tracks.js | 2 +- creator-node/test/lib/reqMock.js | 3 +- 9 files changed, 35 insertions(+), 57 deletions(-) diff --git a/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js index a1c4ad457d3..4874026f612 100644 --- a/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js +++ b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js @@ -69,9 +69,6 @@ module.exports = { COMMIT; `) - // TODO - add a primary key to Tracks (blockchainId:clock) - // TODO - add a primary key to AudiusUsers (blockchainId:clock) - // TODO - remove Files unique constraint since pkey does that }, down: (queryInterface, Sequelize) => { diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index 5eb161c36d0..7f2dd2fabd2 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -9,18 +9,21 @@ module.exports = { up: async (queryInterface, Sequelize) => { const transaction = await queryInterface.sequelize.transaction() - // TODO remove - Add 'clock2' column to all 4 data tables - await addClock2Column(queryInterface, Sequelize, transaction, true) - // Add 'clock' column to all 4 tables await addClockColumn(queryInterface, Sequelize, transaction, false) // Add composite uniqueness constraint on (cnodeUserUUID, clock) to all Content Tables - await addUniquenessConstraints(queryInterface, Sequelize, transaction) + await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) // Create Clock table await createClockRecordsTable(queryInterface, Sequelize, transaction) + // await addCompositePrimaryKeys(queryInterface, Sequelize, transaction) + + // TODO - add a primary key to Tracks (blockchainId:clock) + // TODO - add a primary key to AudiusUsers (blockchainId:clock) + // TODO - remove Files unique constraint since pkey does that + await transaction.commit() }, @@ -71,37 +74,14 @@ async function addClockColumn (queryInterface, Sequelize, transaction, allowNull }, { transaction }) } -async function addClock2Column (queryInterface, Sequelize, transaction, allowNull) { - await queryInterface.addColumn('CNodeUsers', 'clock2', { - type: Sequelize.INTEGER, - unique: false, - allowNull - }, { transaction }) - await queryInterface.addColumn('AudiusUsers', 'clock2', { - type: Sequelize.INTEGER, - unique: false, - allowNull - }, { transaction }) - await queryInterface.addColumn('Tracks', 'clock2', { - type: Sequelize.INTEGER, - unique: false, - allowNull - }, { transaction }) - await queryInterface.addColumn('Files', 'clock2', { - type: Sequelize.INTEGER, - unique: false, - allowNull - }, { transaction }) -} - // Add uniqueness constraint on composite (cnodeUserUUId, clock) to Content Tables -async function addUniquenessConstraints (queryInterface, Sequelize, transaction) { +async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) { await queryInterface.addConstraint( 'AudiusUsers', { type: 'UNIQUE', fields: ['cnodeUserUUID', 'clock'], - name: 'AudiusUsers_unique_constraint_(cnodeUserUUID,clock)', + name: 'AudiusUsers_unique_(cnodeUserUUID,clock)', transaction } ) @@ -110,7 +90,7 @@ async function addUniquenessConstraints (queryInterface, Sequelize, transaction) { type: 'UNIQUE', fields: ['cnodeUserUUID', 'clock'], - name: 'Tracks_unique_constraint_(cnodeUserUUID,clock)', + name: 'Tracks_unique_(cnodeUserUUID,clock)', transaction } ) @@ -119,7 +99,7 @@ async function addUniquenessConstraints (queryInterface, Sequelize, transaction) { type: 'UNIQUE', fields: ['cnodeUserUUID', 'clock'], - name: 'Files_unique_constraint_(cnodeUserUUID,clock)', + name: 'Files_unique_(cnodeUserUUID,clock)', transaction } ) diff --git a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js index 6694f37552a..7c09cfb8fb7 100644 --- a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js +++ b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js @@ -8,13 +8,12 @@ function readOnlyMiddleware (req, res, next) { const isReadOnlyMode = config.get('isReadOnlyMode') const method = req.method const canProceed = readOnlyMiddlewareHelper(isReadOnlyMode, method) - + if (!canProceed) return sendResponse(req, res, errorResponseServerError('Server is in read-only mode at the moment')) next() } /** - * * @param {Boolean} isReadOnlyMode From config.get('isReadOnlyMode') * @param {String} method REST method for this request eg. POST, GET * @returns {Boolean} returns true if the request can proceed. eg GET in read only or any request in non read-only mode diff --git a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js index 5b88dcab6da..86dd28bbca9 100644 --- a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js +++ b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.test.js @@ -4,7 +4,7 @@ const config = require('../../config') const { resFactory, loggerFactory } = require('../../../test/lib/reqMock') describe('Test read-only middleware', function () { - beforeEach(function() { + beforeEach(function () { config.reset('isReadOnlyMode') }) @@ -14,8 +14,7 @@ describe('Test read-only middleware', function () { config.set('isReadOnlyMode', isReadOnlyMode) let nextCalled = false - - readOnlyMiddleware({ method }, {}, function() { + readOnlyMiddleware({ method }, {}, function () { nextCalled = true }) @@ -36,7 +35,7 @@ describe('Test read-only middleware', function () { } const res = resFactory() - readOnlyMiddleware(req, res, function() { + readOnlyMiddleware(req, res, function () { nextCalled = true }) @@ -51,7 +50,7 @@ describe('Test read-only middleware', function () { config.set('isReadOnlyMode', isReadOnlyMode) let nextCalled = false - readOnlyMiddleware({ method }, {}, function() { + readOnlyMiddleware({ method }, {}, function () { nextCalled = true }) diff --git a/creator-node/src/models/track.js b/creator-node/src/models/track.js index 35539dcecbc..c812d551449 100644 --- a/creator-node/src/models/track.js +++ b/creator-node/src/models/track.js @@ -16,7 +16,7 @@ module.exports = (sequelize, DataTypes) => { }, blockchainId: { type: DataTypes.BIGINT, - allowNull: false, + allowNull: false }, coverArtFileUUID: { type: DataTypes.UUID, diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index dd32f41f5bd..f61509377dc 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -105,7 +105,7 @@ module.exports = function (app) { await updateClockInCNodeUserAndClockRecords(req, 'AudiusUser', transaction) // Insert new audiusUser entry to DB - const audiusUser = await models.AudiusUser.create({ + await models.AudiusUser.create({ cnodeUserUUID, metadataFileUUID, metadataJSON, diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 6ba1f8affab..8b27ed90cea 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -58,16 +58,20 @@ module.exports = function (app) { }) audiusUsers.forEach(audiusUser => { - cnodeUsersDict[audiusUserDictObj['cnodeUserUUID']]['audiusUsers'].push(audiusUser.toJSON()) + const audiusUserDictObj = audiusUser.toJSON() + cnodeUsersDict[audiusUserDictObj['cnodeUserUUID']]['audiusUsers'].push(audiusUserDictObj) }) tracks.forEach(track => { - cnodeUsersDict[trackDictObj['cnodeUserUUID']]['tracks'].push(track.toJSON()) + let trackDictObj = track.toJSON() + cnodeUsersDict[trackDictObj['cnodeUserUUID']]['tracks'].push(trackDictObj) }) files.forEach(file => { - cnodeUsersDict[fileDictObj['cnodeUserUUID']]['files'].push(file.toJSON()) + let fileDictObj = file.toJSON() + cnodeUsersDict[fileDictObj['cnodeUserUUID']]['files'].push(fileDictObj) }) clockRecords.forEach(clockRecord => { - cnodeUsersDict[clockRecordDictObj['cnodeUserUUID']]['clockRecords'].push(clockRecord.toJSON()) + let clockRecordDictObj = clockRecord.toJSON() + cnodeUsersDict[clockRecordDictObj['cnodeUserUUID']]['clockRecords'].push(clockRecordDictObj) }) // Expose ipfs node's peer ID. @@ -252,7 +256,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync transaction }) req.logger.info(redisKey, `numAudiusUsersDeleted ${numAudiusUsersDeleted}`) - + // TrackFiles must be deleted before associated Tracks can be deleted. const numTrackFilesDeleted = await models.File.destroy({ where: { @@ -262,20 +266,20 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync transaction }) req.logger.info(redisKey, `numTrackFilesDeleted ${numTrackFilesDeleted}`) - + const numTracksDeleted = await models.Track.destroy({ where: { cnodeUserUUID }, transaction }) req.logger.info(redisKey, `numTracksDeleted ${numTracksDeleted}`) - + // Delete all remaining files (image / metadata files). const numNonTrackFilesDeleted = await models.File.destroy({ where: { cnodeUserUUID }, transaction }) req.logger.info(redisKey, `numNonTrackFilesDeleted ${numNonTrackFilesDeleted}`) - + const numClockRecordsDeleted = await models.ClockRecord.destroy({ where: { cnodeUserUUID }, transaction @@ -283,7 +287,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync req.logger.info(redisKey, `numClockRecordsDeleted ${numClockRecordsDeleted}`) // Delete cnodeUser entry - await cnodeUser.destroy({transaction}) + await cnodeUser.destroy({ transaction }) req.logger.info(redisKey, `deleted cnodeUserEntry`) } @@ -379,13 +383,13 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync })), { transaction }) req.logger.info('saved all audiususer data to db') - await t.commit() + await transaction.commit() req.logger.info(redisKey, `Transaction successfully committed for cnodeUserUUID ${fetchedCnodeUserUUID}`) redisKey = redisClient.getNodeSyncRedisKey(fetchedWalletPublicKey) await redisLock.removeLock(redisKey) } catch (e) { req.logger.error(redisKey, `Transaction failed for cnodeUserUUID ${fetchedCnodeUserUUID}`, e) - await t.rollback() + await transaction.rollback() redisKey = redisClient.getNodeSyncRedisKey(fetchedWalletPublicKey) await redisLock.removeLock(redisKey) throw new Error(e) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 60e7aea4a26..b26b76712ef 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -411,7 +411,7 @@ module.exports = function (app) { transaction } ) - logger.error(`\n\n\nnumAffectedRows: ${numAffectedRows}`) + logger.error(`\n\n\nnumAffectedRows: ${numAffectedRows}`) if (parseInt(numAffectedRows, 10) !== trackSegmentCIDs.length) { throw new Error('Failed to associate files for every track segment CID.') } diff --git a/creator-node/test/lib/reqMock.js b/creator-node/test/lib/reqMock.js index 26c66ddffc9..7e83dbb857b 100644 --- a/creator-node/test/lib/reqMock.js +++ b/creator-node/test/lib/reqMock.js @@ -40,5 +40,4 @@ const loggerFactory = () => { return logger } - -module.exports = { resFactory, loggerFactory } \ No newline at end of file +module.exports = { resFactory, loggerFactory } From e07c345e83be578ed1109cd4b22476cbc80a8076 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 18 Sep 2020 15:53:38 +0000 Subject: [PATCH 18/53] Fixed migration + models --- .../migrations/20200918150546-add-clock.js | 41 ++++++++++++++----- creator-node/src/models/audiususer.js | 21 +++++----- creator-node/src/models/cNodeUser.js | 3 -- creator-node/src/models/file.js | 3 -- creator-node/src/models/track.js | 21 +++++----- 5 files changed, 51 insertions(+), 38 deletions(-) diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index 7f2dd2fabd2..36248705d13 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -12,16 +12,17 @@ module.exports = { // Add 'clock' column to all 4 tables await addClockColumn(queryInterface, Sequelize, transaction, false) - // Add composite uniqueness constraint on (cnodeUserUUID, clock) to all Content Tables + // Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables + // (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) + await addCompositePrimaryKeysToAudiusUsersAndTracks(queryInterface, Sequelize, transaction) + + // Add composite unique constraint on (blockchainId, clock) to AudiusUsers and Tracks + // Add composite unique constraint on (cnodeUserUUID, clock) to Files await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) // Create Clock table await createClockRecordsTable(queryInterface, Sequelize, transaction) - // await addCompositePrimaryKeys(queryInterface, Sequelize, transaction) - - // TODO - add a primary key to Tracks (blockchainId:clock) - // TODO - add a primary key to AudiusUsers (blockchainId:clock) // TODO - remove Files unique constraint since pkey does that await transaction.commit() @@ -74,14 +75,34 @@ async function addClockColumn (queryInterface, Sequelize, transaction, allowNull }, { transaction }) } -// Add uniqueness constraint on composite (cnodeUserUUId, clock) to Content Tables +async function addCompositePrimaryKeysToAudiusUsersAndTracks (queryInterface, Sequelize, transaction) { + await queryInterface.addConstraint( + 'AudiusUsers', + { + type: 'PRIMARY KEY', + fields: ['cnodeUserUUID', 'clock'], + name: 'AudiusUsers_primary_key_(cnodeUserUUID,clock)', + transaction + } + ) + await queryInterface.addConstraint( + 'Tracks', + { + type: 'PRIMARY KEY', + fields: ['cnodeUserUUID', 'clock'], + name: 'Tracks_primary_key_(cnodeUserUUID,clock)', + transaction + } + ) +} + async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) { await queryInterface.addConstraint( 'AudiusUsers', { type: 'UNIQUE', - fields: ['cnodeUserUUID', 'clock'], - name: 'AudiusUsers_unique_(cnodeUserUUID,clock)', + fields: ['blockchainId', 'clock'], + name: 'AudiusUsers_unique_(blockchainId,clock)', transaction } ) @@ -89,8 +110,8 @@ async function addCompositeUniqueConstraints (queryInterface, Sequelize, transac 'Tracks', { type: 'UNIQUE', - fields: ['cnodeUserUUID', 'clock'], - name: 'Tracks_unique_(cnodeUserUUID,clock)', + fields: ['blockchainId', 'clock'], + name: 'Tracks_unique_(blockchainId,clock)', transaction } ) diff --git a/creator-node/src/models/audiususer.js b/creator-node/src/models/audiususer.js index 77cbdde8cd1..49685ae75c0 100644 --- a/creator-node/src/models/audiususer.js +++ b/creator-node/src/models/audiususer.js @@ -1,12 +1,18 @@ 'use strict' module.exports = (sequelize, DataTypes) => { const AudiusUser = sequelize.define('AudiusUser', { - blockchainId: { - type: DataTypes.BIGINT, - allowNull: false - }, cnodeUserUUID: { type: DataTypes.UUID, + primaryKey: true, + allowNull: false + }, + clock: { + type: DataTypes.INTEGER, + primaryKey: true, + allowNull: false + }, + blockchainId: { + type: DataTypes.BIGINT, allowNull: false }, metadataFileUUID: { @@ -24,13 +30,6 @@ module.exports = (sequelize, DataTypes) => { profilePicFileUUID: { type: DataTypes.UUID, allowNull: true - }, - clock: { - type: DataTypes.INTEGER, - allowNull: false - }, - clock2: { - type: DataTypes.INTEGER } }, {}) AudiusUser.associate = function (models) { diff --git a/creator-node/src/models/cNodeUser.js b/creator-node/src/models/cNodeUser.js index fb2151422fd..bcdf0ca1bb1 100644 --- a/creator-node/src/models/cNodeUser.js +++ b/creator-node/src/models/cNodeUser.js @@ -25,9 +25,6 @@ module.exports = (sequelize, DataTypes) => { clock: { type: DataTypes.INTEGER, allowNull: false - }, - clock2: { - type: DataTypes.INTEGER } }, {}) diff --git a/creator-node/src/models/file.js b/creator-node/src/models/file.js index 6d484b0ee78..4c7479531d3 100644 --- a/creator-node/src/models/file.js +++ b/creator-node/src/models/file.js @@ -53,9 +53,6 @@ module.exports = (sequelize, DataTypes) => { clock: { type: DataTypes.INTEGER, allowNull: false - }, - clock2: { - type: DataTypes.INTEGER } }, { indexes: [ diff --git a/creator-node/src/models/track.js b/creator-node/src/models/track.js index c812d551449..acd9af8cdf2 100644 --- a/creator-node/src/models/track.js +++ b/creator-node/src/models/track.js @@ -4,6 +4,16 @@ module.exports = (sequelize, DataTypes) => { const Track = sequelize.define('Track', { cnodeUserUUID: { type: DataTypes.UUID, + primaryKey: true, + allowNull: false + }, + clock: { + type: DataTypes.INTEGER, + primaryKey: true, + allowNull: false + }, + blockchainId: { + type: DataTypes.BIGINT, allowNull: false }, metadataFileUUID: { @@ -14,20 +24,9 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.JSONB, allowNull: false }, - blockchainId: { - type: DataTypes.BIGINT, - allowNull: false - }, coverArtFileUUID: { type: DataTypes.UUID, allowNull: true - }, - clock: { - type: DataTypes.INTEGER, - allowNull: false - }, - clock2: { - type: DataTypes.INTEGER } }, {}) From b363aaa5ce96f7d898c35d7e86b9ebfc0ca02ebd Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 18 Sep 2020 15:55:17 +0000 Subject: [PATCH 19/53] Remove old comment --- creator-node/sequelize/migrations/20200918150546-add-clock.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index 36248705d13..8e56e85c61e 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -23,8 +23,6 @@ module.exports = { // Create Clock table await createClockRecordsTable(queryInterface, Sequelize, transaction) - // TODO - remove Files unique constraint since pkey does that - await transaction.commit() }, From a026033a8eb2b2b2b0b2a92f40e25ee3fcd892cb Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 18 Sep 2020 16:32:05 +0000 Subject: [PATCH 20/53] new dbManager class + working in audiusUsers --- .../src/{clockManager.js => dbManager.js} | 38 +++++++++++-------- creator-node/src/routes/audiusUsers.js | 23 +++++------ creator-node/test/audiusUsers.test.js | 6 +-- 3 files changed, 36 insertions(+), 31 deletions(-) rename creator-node/src/{clockManager.js => dbManager.js} (52%) diff --git a/creator-node/src/clockManager.js b/creator-node/src/dbManager.js similarity index 52% rename from creator-node/src/clockManager.js rename to creator-node/src/dbManager.js index 34df0c92a00..726019760f7 100644 --- a/creator-node/src/clockManager.js +++ b/creator-node/src/dbManager.js @@ -1,6 +1,9 @@ const models = require('./models') const sequelize = models.sequelize +// TODO - consider moving to inside /src/models +// TODO - turn into class + // TODO consider adding hook to ensure write op can never set clockVal to anything <= current // TODO - any further constraint enforcement needed? @@ -8,14 +11,21 @@ const sequelize = models.sequelize // - Clocks table has unique constraint // - dataTables have unique C on (userId + clock) -const updateClockInCNodeUserAndClockRecords = async (req, sourceTable, transaction) => { - const cnodeUserUUID = req.session.cnodeUserUUID +// source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause +const _getSelectCNodeUserClockSubqueryLiteral = (cnodeUserUUID) => { + const subquery = sequelize.dialect.QueryGenerator.selectQuery('CNodeUsers', { + attributes: ['clock'], + where: { cnodeUserUUID } + }).slice(0, -1) // removes trailing ';' + return sequelize.literal(`(${subquery})`) +} +const createNewFileRecord = async (queryObj, cnodeUserUUID, transaction) => { // Increment CNodeUser.clock value by 1 await models.CNodeUser.increment( 'clock', /* fields */ { - where: { cnodeUserUUID: req.session.cnodeUserUUID }, + where: { cnodeUserUUID }, by: 1, transaction } @@ -23,21 +33,19 @@ const updateClockInCNodeUserAndClockRecords = async (req, sourceTable, transacti // Add row in ClockRecords table using clock value from CNodeUser await models.ClockRecord.create({ - cnodeUserUUID: req.session.cnodeUserUUID, - clock: sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`), - sourceTable + cnodeUserUUID, + clock: _getSelectCNodeUserClockSubqueryLiteral(cnodeUserUUID), + sourceTable: 'File' }, { transaction }) -} -// source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause -const selectCNodeUserClockSubquery = (cnodeUserUUID) => { - return sequelize.dialect.QueryGenerator.selectQuery('CNodeUsers', { - attributes: ['clock'], - where: { cnodeUserUUID } - }).slice(0, -1) // removes trailing ';' + // Add row in Files table with queryObj + queryObj.clock = _getSelectCNodeUserClockSubqueryLiteral(cnodeUserUUID) + queryObj.cnodeUserUUID = cnodeUserUUID + const file = await models.File.create(queryObj, { transaction }) + + return file.dataValues } module.exports = { - updateClockInCNodeUserAndClockRecords, - selectCNodeUserClockSubquery + createNewFileRecord } diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index f61509377dc..db8c160ffac 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -6,7 +6,7 @@ const { saveFileFromBufferToIPFSAndDisk } = require('../fileManager') const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers') const { getFileUUIDForImageCID } = require('../utils') const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares') -const { updateClockInCNodeUserAndClockRecords, selectCNodeUserClockSubquery } = require('../clockManager') +const dbManager = require('../dbManager') const { logger } = require('../logging') module.exports = function (app) { @@ -20,7 +20,7 @@ module.exports = function (app) { const cnodeUserUUID = req.session.cnodeUserUUID // Save file from buffer to IPFS and disk - // TODO simplify + // TODO simplify (object destructuring?) let multihash, dstPath try { const resp = await saveFileFromBufferToIPFSAndDisk(req, metadataBuffer) @@ -34,17 +34,14 @@ module.exports = function (app) { const transaction = await models.sequelize.transaction() let fileUUID try { - await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) - - fileUUID = (await models.File.create({ - cnodeUserUUID, + const createFileQueryObj = { multihash, sourceFile: req.fileName, storagePath: dstPath, - type: 'metadata', // TODO - replace with models enum - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }, { transaction }) - ).dataValues.fileUUID + type: 'metadata' // TODO - replace with models enum + } + const file = await dbManager.createNewFileRecord(createFileQueryObj, cnodeUserUUID, transaction) + fileUUID = file.fileUUID await transaction.commit() } catch (e) { @@ -102,7 +99,7 @@ module.exports = function (app) { try { logger.info(`beginning audiusUsers DB transactions`) - await updateClockInCNodeUserAndClockRecords(req, 'AudiusUser', transaction) + // await updateClockInCNodeUserAndClockRecords(req, 'AudiusUser', transaction) // Insert new audiusUser entry to DB await models.AudiusUser.create({ @@ -111,8 +108,8 @@ module.exports = function (app) { metadataJSON, blockchainId: blockchainUserId, coverArtFileUUID, - profilePicFileUUID, - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) + profilePicFileUUID + // clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) }, { transaction, returning: true }) // Update cnodeUser's latestBlockNumber and clock diff --git a/creator-node/test/audiusUsers.test.js b/creator-node/test/audiusUsers.test.js index 274d80696b0..b225a80f0cf 100644 --- a/creator-node/test/audiusUsers.test.js +++ b/creator-node/test/audiusUsers.test.js @@ -16,7 +16,7 @@ const { getIPFSMock } = require('./lib/ipfsMock') const { getLibsMock } = require('./lib/libsMock') const { sortKeys } = require('../src/apiHelpers') -describe('test AudiusUsers', function () { +describe.only('test AudiusUsers', function () { let app, server, session, ipfsMock, libsMock // Will need a '.' in front of storagePath to look at current dir @@ -46,7 +46,7 @@ describe('test AudiusUsers', function () { await server.close() }) - it('creates Audius user', async function () { + it.only('creates Audius user', async function () { const metadata = { test: 'field1' } ipfsMock.add.twice().withArgs(Buffer.from(JSON.stringify(metadata))) ipfsMock.pin.add.once().withArgs('testCIDLink') @@ -57,7 +57,7 @@ describe('test AudiusUsers', function () { .send({ metadata }) .expect(200) - if (resp.body.metadataMultihash !== 'testCIDLink') { + if (resp.body.metadataMultihash !== 'testCIDLink' || !resp.body.metadataFileUUID) { throw new Error('invalid return data') } }) From da6a0126b8369262658dd69dfb1069e620ef4235 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 18 Sep 2020 17:02:29 +0000 Subject: [PATCH 21/53] Changed DBManager structure + all AudiusUSers tests working --- creator-node/src/dbManager.js | 107 ++++++++++++++++--------- creator-node/src/routes/audiusUsers.js | 22 ++--- creator-node/test/audiusUsers.test.js | 4 +- 3 files changed, 80 insertions(+), 53 deletions(-) diff --git a/creator-node/src/dbManager.js b/creator-node/src/dbManager.js index 726019760f7..893fd1afcb3 100644 --- a/creator-node/src/dbManager.js +++ b/creator-node/src/dbManager.js @@ -1,18 +1,79 @@ const models = require('./models') const sequelize = models.sequelize -// TODO - consider moving to inside /src/models -// TODO - turn into class +/** +TODO - consider moving to inside /src/models -// TODO consider adding hook to ensure write op can never set clockVal to anything <= current +TODO consider adding hook to ensure write op can never set clockVal to anything <= current -// TODO - any further constraint enforcement needed? -// - DataTables all FK to Clocks table -// - Clocks table has unique constraint +TODO - any further constraint enforcement needed? +- DataTables all FK to Clocks table + - Clocks table has unique constraint // - dataTables have unique C on (userId + clock) + */ -// source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause -const _getSelectCNodeUserClockSubqueryLiteral = (cnodeUserUUID) => { +class DBManager { + /** + * Given file insert query object and cnodeUserUUID, inserts new file record in DB + * and handles all required clock management. + * Steps: + * 1. increments cnodeUser clock value + * 2. insert new ClockRecord entry with new clock value + * 3. insert new File entry with queryObj and new clock value + * + * After initial increment, clock values are read as subquery without reading into JS to guarantee atomicity + * + * TODO - flesh out jsdoc + * @param {*} queryObj - object with file insert data + * @param {*} cnodeUserUUID + * @param {*} transaction + * + * TODO - returns + */ + static async createNewDataRecord (queryObj, cnodeUserUUID, sourceTable, transaction) { + // Increment CNodeUser.clock value by 1 + await models.CNodeUser.increment('clock', { + where: { cnodeUserUUID }, + by: 1, + transaction + }) + + const selectCNodeUserClockSubqueryLiteral = _getSelectCNodeUserClockSubqueryLiteral(cnodeUserUUID) + + // Add row in ClockRecords table using clock value from CNodeUser + await models.ClockRecord.create({ + cnodeUserUUID, + clock: selectCNodeUserClockSubqueryLiteral, + sourceTable + }, { transaction }) + + // Add cnodeUserUUID + clock value to queryObj + queryObj.cnodeUserUUID = cnodeUserUUID + queryObj.clock = selectCNodeUserClockSubqueryLiteral + + // Create new File entry with queryObj + const modelsInstance = _getModelsInstance(sourceTable) + const file = await modelsInstance.create(queryObj, { transaction }) + + return file.dataValues + } +} + +const _SourceTableToModelInstanceMap = { + 'AudiusUser': models.AudiusUser, + 'Track': models.Track, + 'File': models.File +} + +function _getModelsInstance (sourceTable) { + return _SourceTableToModelInstanceMap[sourceTable] +} + +/** + * @dev source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause + * @param {*} cnodeUserUUID + */ +function _getSelectCNodeUserClockSubqueryLiteral (cnodeUserUUID) { const subquery = sequelize.dialect.QueryGenerator.selectQuery('CNodeUsers', { attributes: ['clock'], where: { cnodeUserUUID } @@ -20,32 +81,4 @@ const _getSelectCNodeUserClockSubqueryLiteral = (cnodeUserUUID) => { return sequelize.literal(`(${subquery})`) } -const createNewFileRecord = async (queryObj, cnodeUserUUID, transaction) => { - // Increment CNodeUser.clock value by 1 - await models.CNodeUser.increment( - 'clock', /* fields */ - { - where: { cnodeUserUUID }, - by: 1, - transaction - } - ) - - // Add row in ClockRecords table using clock value from CNodeUser - await models.ClockRecord.create({ - cnodeUserUUID, - clock: _getSelectCNodeUserClockSubqueryLiteral(cnodeUserUUID), - sourceTable: 'File' - }, { transaction }) - - // Add row in Files table with queryObj - queryObj.clock = _getSelectCNodeUserClockSubqueryLiteral(cnodeUserUUID) - queryObj.cnodeUserUUID = cnodeUserUUID - const file = await models.File.create(queryObj, { transaction }) - - return file.dataValues -} - -module.exports = { - createNewFileRecord -} +module.exports = DBManager diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index db8c160ffac..b4d7dd20aed 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -6,8 +6,8 @@ const { saveFileFromBufferToIPFSAndDisk } = require('../fileManager') const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers') const { getFileUUIDForImageCID } = require('../utils') const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares') -const dbManager = require('../dbManager') -const { logger } = require('../logging') +const DBManager = require('../dbManager') +// const { logger } = require('../logging') module.exports = function (app) { /** @@ -40,7 +40,7 @@ module.exports = function (app) { storagePath: dstPath, type: 'metadata' // TODO - replace with models enum } - const file = await dbManager.createNewFileRecord(createFileQueryObj, cnodeUserUUID, transaction) + const file = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, 'File', transaction) fileUUID = file.fileUUID await transaction.commit() @@ -95,24 +95,18 @@ module.exports = function (app) { } const transaction = await models.sequelize.transaction() - try { - logger.info(`beginning audiusUsers DB transactions`) - - // await updateClockInCNodeUserAndClockRecords(req, 'AudiusUser', transaction) - - // Insert new audiusUser entry to DB - await models.AudiusUser.create({ - cnodeUserUUID, + const createAudiusUserQueryObj = { metadataFileUUID, metadataJSON, blockchainId: blockchainUserId, coverArtFileUUID, profilePicFileUUID - // clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }, { transaction, returning: true }) + } + await DBManager.createNewDataRecord(createAudiusUserQueryObj, cnodeUserUUID, 'AudiusUser', transaction) - // Update cnodeUser's latestBlockNumber and clock + // Update cnodeUser.latestBlockNumber + // TODO - can this be deprecated with new clock logic? await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction }) await transaction.commit() diff --git a/creator-node/test/audiusUsers.test.js b/creator-node/test/audiusUsers.test.js index b225a80f0cf..27876f5f153 100644 --- a/creator-node/test/audiusUsers.test.js +++ b/creator-node/test/audiusUsers.test.js @@ -16,7 +16,7 @@ const { getIPFSMock } = require('./lib/ipfsMock') const { getLibsMock } = require('./lib/libsMock') const { sortKeys } = require('../src/apiHelpers') -describe.only('test AudiusUsers', function () { +describe('test AudiusUsers', function () { let app, server, session, ipfsMock, libsMock // Will need a '.' in front of storagePath to look at current dir @@ -46,7 +46,7 @@ describe.only('test AudiusUsers', function () { await server.close() }) - it.only('creates Audius user', async function () { + it('creates Audius user', async function () { const metadata = { test: 'field1' } ipfsMock.add.twice().withArgs(Buffer.from(JSON.stringify(metadata))) ipfsMock.pin.add.once().withArgs('testCIDLink') From 484b458d24df461a8202a18716601a25f6c9d11d Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 18 Sep 2020 19:15:41 +0000 Subject: [PATCH 22/53] All routes updated to use new DBManager + all tests working --- creator-node/src/routes/audiusUsers.js | 1 - creator-node/src/routes/files.js | 22 ++++---- creator-node/src/routes/tracks.js | 52 +++++++------------ .../test/{tracks.js => tracks.test.js} | 0 creator-node/test/{users.js => users.test.js} | 0 5 files changed, 29 insertions(+), 46 deletions(-) rename creator-node/test/{tracks.js => tracks.test.js} (100%) rename creator-node/test/{users.js => users.test.js} (100%) diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index b4d7dd20aed..494c627b3b2 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -7,7 +7,6 @@ const { handleResponse, successResponse, errorResponseBadRequest, errorResponseS const { getFileUUIDForImageCID } = require('../utils') const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares') const DBManager = require('../dbManager') -// const { logger } = require('../logging') module.exports = function (app) { /** diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js index 2f17b2cf9b1..43f0d924625 100644 --- a/creator-node/src/routes/files.js +++ b/creator-node/src/routes/files.js @@ -22,7 +22,7 @@ const { authMiddleware, syncLockMiddleware, triggerSecondarySyncs } = require('. const { getIPFSPeerId, ipfsSingleByteCat, ipfsStat } = require('../utils') const ImageProcessingQueue = require('../ImageProcessingQueue') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') -const { updateClockInCNodeUserAndClockRecords, selectCNodeUserClockSubquery } = require('../clockManager') +const DBManager = require('../dbManager') /** * Helper method to stream file from file system on creator node @@ -290,30 +290,26 @@ module.exports = function (app) { const transaction = await models.sequelize.transaction() try { // Record dir file entry in DB - await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) - await models.File.create({ - cnodeUserUUID, + const createDirFileQueryObj = { multihash: resizeResp.dir.dirCID, sourceFile: null, storagePath: resizeResp.dir.dirDestPath, - type: 'dir', // TODO - replace with models enum - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }, { transaction }) + type: 'dir' // TODO - replace with models enum + } + await DBManager.createNewDataRecord(createDirFileQueryObj, cnodeUserUUID, 'File', transaction) // Record all image res file entries in DB // Must be written sequentially to ensure clock values are correctly incremented and populated for (const file of resizeResp.files) { - await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) - await models.File.create({ - cnodeUserUUID, + const createImageFileQueryObj = { multihash: file.multihash, sourceFile: file.sourceFile, storagePath: file.storagePath, type: 'image', // TODO - replace with models enum dirMultihash: resizeResp.dir.dirCID, - fileName: file.sourceFile.split('/').slice(-1)[0], - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }) + fileName: file.sourceFile.split('/').slice(-1)[0] + } + await DBManager.createNewDataRecord(createImageFileQueryObj, cnodeUserUUID, 'File', transaction) } req.logger.info(`route time = ${Date.now() - routestart}`) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index b26b76712ef..e412b1b57c1 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -14,7 +14,7 @@ const { getCID } = require('./files') const { decode } = require('../hashids.js') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') const { logger } = require('../logging.js') -const { updateClockInCNodeUserAndClockRecords, selectCNodeUserClockSubquery } = require('../clockManager') +const DBManager = require('../dbManager') module.exports = function (app) { /** @@ -118,29 +118,25 @@ module.exports = function (app) { let transcodeFileUUID try { // Record transcode file entry in DB - await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) - transcodeFileUUID = (await models.File.create({ - cnodeUserUUID, + const createTranscodeFileQueryObj = { multihash: transcodeFileIPFSResp.multihash, sourceFile: req.fileName, storagePath: transcodeFileIPFSResp.dstPath, - type: 'copy320', // TODO - replace with models enum - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }, { transaction }) - ).dataValues.fileUUID + type: 'copy320' // TODO - replace with models enum + } + const file = await DBManager.createNewDataRecord(createTranscodeFileQueryObj, cnodeUserUUID, 'File', transaction) + transcodeFileUUID = file.fileUUID // Record all segment file entries in DB // Must be written sequentially to ensure clock values are correctly incremented and populated for (const { multihash, dstPath } of segmentFileIPFSResps) { - await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) - await models.File.create({ - cnodeUserUUID, + const createSegmentFileQueryObj = { multihash, sourceFile: req.fileName, storagePath: dstPath, - type: 'track', // TODO - replace with models enum - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }, { transaction }) + type: 'track' // TODO - replace with models enum + } + await DBManager.createNewDataRecord(createSegmentFileQueryObj, cnodeUserUUID, 'File', transaction) } await transaction.commit() @@ -246,17 +242,14 @@ module.exports = function (app) { const transaction = await models.sequelize.transaction() let fileUUID try { - await updateClockInCNodeUserAndClockRecords(req, 'File', transaction) - - fileUUID = (await models.File.create({ - cnodeUserUUID, + const createFileQueryObj = { multihash, sourceFile: req.fileName, storagePath: dstPath, - type: 'metadata', // TODO - replace with models enum - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }, { transaction }) - ).dataValues.fileUUID + type: 'metadata' // TODO - replace with models enum + } + const file = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, 'File', transaction) + fileUUID = file.fileUUID await transaction.commit() } catch (e) { @@ -323,7 +316,6 @@ module.exports = function (app) { const existingTrackEntry = await models.Track.findOne({ where: { cnodeUserUUID, - // metadataFileUUID, blockchainId: blockchainTrackId, coverArtFileUUID }, @@ -331,18 +323,14 @@ module.exports = function (app) { transaction }) - await updateClockInCNodeUserAndClockRecords(req, 'Track', transaction) - - // Insert new track entry on db (for track update, a new entry is still created with incremented clock val) - const track = await models.Track.create({ - cnodeUserUUID, + // Insert track entry in DB + const createTrackQueryObj = { metadataFileUUID, metadataJSON, blockchainId: blockchainTrackId, - coverArtFileUUID, - clock: models.sequelize.literal(`(${selectCNodeUserClockSubquery(cnodeUserUUID)})`) - }, { transaction } - ) + coverArtFileUUID + } + const track = await DBManager.createNewDataRecord(createTrackQueryObj, cnodeUserUUID, 'Track', transaction) /** * Associate matching transcode & segment files on DB with new/updated track diff --git a/creator-node/test/tracks.js b/creator-node/test/tracks.test.js similarity index 100% rename from creator-node/test/tracks.js rename to creator-node/test/tracks.test.js diff --git a/creator-node/test/users.js b/creator-node/test/users.test.js similarity index 100% rename from creator-node/test/users.js rename to creator-node/test/users.test.js From 14ec98c14fd22f6ba3ddb40c96c8555e4e2fd3e0 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Sun, 20 Sep 2020 12:09:55 -0400 Subject: [PATCH 23/53] backfill vector clock migration + nodesync changes for db only migration working --- .../migrations/20200918150546-add-clock.js | 12 +- creator-node/src/clockManager.test.js | 1 - .../readOnly/readOnlyMiddleware.js | 15 ++- creator-node/src/routes/nodeSync.js | 98 ++++++++++----- creator-node/src/routes/vectorClock.js | 117 ++++++++++++++++++ 5 files changed, 202 insertions(+), 41 deletions(-) create mode 100644 creator-node/src/routes/vectorClock.js diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index 8e56e85c61e..f933e72b522 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -10,18 +10,18 @@ module.exports = { const transaction = await queryInterface.sequelize.transaction() // Add 'clock' column to all 4 tables - await addClockColumn(queryInterface, Sequelize, transaction, false) + await addClockColumn(queryInterface, Sequelize, transaction, true) - // Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables - // (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) - await addCompositePrimaryKeysToAudiusUsersAndTracks(queryInterface, Sequelize, transaction) + // Create Clock table + await createClockRecordsTable(queryInterface, Sequelize, transaction) // Add composite unique constraint on (blockchainId, clock) to AudiusUsers and Tracks // Add composite unique constraint on (cnodeUserUUID, clock) to Files await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) - // Create Clock table - await createClockRecordsTable(queryInterface, Sequelize, transaction) + // Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables + // (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) + // await addCompositePrimaryKeysToAudiusUsersAndTracks(queryInterface, Sequelize, transaction) await transaction.commit() }, diff --git a/creator-node/src/clockManager.test.js b/creator-node/src/clockManager.test.js index f9c95625a92..411a3447eba 100644 --- a/creator-node/src/clockManager.test.js +++ b/creator-node/src/clockManager.test.js @@ -39,7 +39,6 @@ describe.skip('Test incrementAndFetchCNodeUserClock', () => { }) }) - it('Sequential increment&Fetch', async () => { // Explicitly pass in incrementBy value const newClockVal1 = await incrementAndFetchCNodeUserClock(req, incrementClockBy) diff --git a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js index 7c09cfb8fb7..23d4a7e28c9 100644 --- a/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js +++ b/creator-node/src/middlewares/readOnly/readOnlyMiddleware.js @@ -1,13 +1,15 @@ const { sendResponse, errorResponseServerError } = require('../../apiHelpers') const config = require('../../config') +const exclusionsRegex = new RegExp(/(vector_clock)/) /** * Middleware to block all non-GET api calls if the server should be in "read-only" mode */ function readOnlyMiddleware (req, res, next) { const isReadOnlyMode = config.get('isReadOnlyMode') const method = req.method - const canProceed = readOnlyMiddlewareHelper(isReadOnlyMode, method) + const url = req.url + const canProceed = readOnlyMiddlewareHelper(isReadOnlyMode, method, url) if (!canProceed) return sendResponse(req, res, errorResponseServerError('Server is in read-only mode at the moment')) next() @@ -18,8 +20,15 @@ function readOnlyMiddleware (req, res, next) { * @param {String} method REST method for this request eg. POST, GET * @returns {Boolean} returns true if the request can proceed. eg GET in read only or any request in non read-only mode */ -function readOnlyMiddlewareHelper (isReadOnlyMode, method) { - if (isReadOnlyMode && method !== 'GET') return false +function readOnlyMiddlewareHelper (isReadOnlyMode, method, url) { + // is an excluded route...let it pass through + if (exclusionsRegex.test(url)) { + return true + } + + if (isReadOnlyMode && method !== 'GET') { + return false + } return true } diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 8b27ed90cea..97b714b197b 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -25,6 +25,7 @@ module.exports = function (app) { */ app.get('/export', handleResponse(async (req, res) => { const walletPublicKeys = req.query.wallet_public_key // array + const dbOnlySync = (req.query.db_only_sync === true || req.query.db_only_sync === 'true') const transaction = await models.sequelize.transaction() try { @@ -78,29 +79,32 @@ module.exports = function (app) { const ipfs = req.app.get('ipfsAPI') const ipfsIDObj = await getIPFSPeerId(ipfs, config) - // Rehydrate files if necessary - for (let i = 0; i < files.length; i += RehydrateIPFSConcurrencyLimit) { - const exportFilesSlice = files.slice(i, i + RehydrateIPFSConcurrencyLimit) - req.logger.info(`Export rehydrateIpfs processing files ${i} to ${i + RehydrateIPFSConcurrencyLimit}`) - // Ensure all relevant files are available through IPFS at export time - await Promise.all(exportFilesSlice.map(async (file) => { - try { - if ( - (file.type === 'track' || file.type === 'metadata' || file.type === 'copy320') || - // to address legacy single-res image rehydration where images are stored directly under its file CID - (file.type === 'image' && file.sourceFile === null) - ) { - await RehydrateIpfsQueue.addRehydrateIpfsFromFsIfNecessaryTask(file.multihash, file.storagePath, { logContext: req.logContext }) - } else if (file.type === 'dir') { - await RehydrateIpfsQueue.addRehydrateIpfsDirFromFsIfNecessaryTask(file.multihash, { logContext: req.logContext }) + if (!dbOnlySync) { + // Rehydrate files if necessary + for (let i = 0; i < files.length; i += RehydrateIPFSConcurrencyLimit) { + const exportFilesSlice = files.slice(i, i + RehydrateIPFSConcurrencyLimit) + req.logger.info(`Export rehydrateIpfs processing files ${i} to ${i + RehydrateIPFSConcurrencyLimit}`) + // Ensure all relevant files are available through IPFS at export time + await Promise.all(exportFilesSlice.map(async (file) => { + try { + if ( + (file.type === 'track' || file.type === 'metadata' || file.type === 'copy320') || + // to address legacy single-res image rehydration where images are stored directly under its file CID + (file.type === 'image' && file.sourceFile === null) + ) { + await RehydrateIpfsQueue.addRehydrateIpfsFromFsIfNecessaryTask(file.multihash, file.storagePath, { logContext: req.logContext }) + } else if (file.type === 'dir') { + await RehydrateIpfsQueue.addRehydrateIpfsDirFromFsIfNecessaryTask(file.multihash, { logContext: req.logContext }) + } + } catch (e) { + req.logger.info(`Export rehydrateIpfs processing files ${i} to ${i + RehydrateIPFSConcurrencyLimit}, ${e}`) } - } catch (e) { - req.logger.info(`Export rehydrateIpfs processing files ${i} to ${i + RehydrateIPFSConcurrencyLimit}, ${e}`) - } - })) + })) + } } return successResponse({ cnodeUsers: cnodeUsersDict, ipfsIDObj }) } catch (e) { + console.error('Error in /export', e) await transaction.rollback() return errorResponseServerError(e.message) } @@ -137,6 +141,35 @@ module.exports = function (app) { return successResponse() })) + // copy the code as the regular sync, just to make sure it's isolated and not called by any other cnode code + // force immediate and dbOnlySync to be true + app.post('/vector_clock_sync', handleResponse(async (req, res) => { + const walletPublicKeys = req.body.wallet // array + const creatorNodeEndpoint = req.body.creator_node_endpoint // string + const immediate = true + // option to sync just the db records as opposed to db records and files on disk, defaults to false + const dbOnlySync = true + + if (!immediate) { + req.logger.info('debounce time', config.get('debounceTime')) + // Debounce nodeysnc op + for (let wallet of walletPublicKeys) { + if (wallet in syncQueue) { + clearTimeout(syncQueue[wallet]) + req.logger.info('clear timeout for', wallet, 'time', Date.now()) + } + syncQueue[wallet] = setTimeout( + async () => _nodesync(req, [wallet], creatorNodeEndpoint, dbOnlySync), + config.get('debounceTime') + ) + req.logger.info('set timeout for', wallet, 'time', Date.now()) + } + } else { + await _nodesync(req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync) + } + return successResponse() + })) + /** Checks if node sync is in progress for wallet. */ app.get('/sync_status/:walletPublicKey', handleResponse(async (req, res) => { const walletPublicKey = req.params.walletPublicKey @@ -179,7 +212,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync method: 'get', baseURL: creatorNodeEndpoint, url: '/export', - params: { wallet_public_key: walletPublicKeys }, + params: { wallet_public_key: walletPublicKeys, db_only_sync: dbOnlySync }, responseType: 'json' }) if (resp.status !== 200) throw new Error(resp.data['error']) @@ -206,20 +239,23 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync } const fetchedWalletPublicKey = fetchedCNodeUser.walletPublicKey let userReplicaSet = [] - try { - const myCnodeEndpoint = await middlewares.getOwnEndpoint(req) - userReplicaSet = await middlewares.getCreatorNodeEndpoints(req, fetchedWalletPublicKey) - // push user metadata node to user's replica set if defined - if (config.get('userMetadataNodeUrl')) userReplicaSet.push(config.get('userMetadataNodeUrl')) + if (!dbOnlySync) { + try { + const myCnodeEndpoint = await middlewares.getOwnEndpoint(req) + userReplicaSet = await middlewares.getCreatorNodeEndpoints(req, fetchedWalletPublicKey) - // filter out current node from user's replica set - userReplicaSet = userReplicaSet.filter(url => url !== myCnodeEndpoint) + // push user metadata node to user's replica set if defined + if (config.get('userMetadataNodeUrl')) userReplicaSet.push(config.get('userMetadataNodeUrl')) - // Spread + set uniq's the array - userReplicaSet = [...new Set(userReplicaSet)] - } catch (e) { - req.logger.error(`Couldn't get user's replica sets, can't use cnode gateways in saveFileForMultihash`) + // filter out current node from user's replica set + userReplicaSet = userReplicaSet.filter(url => url !== myCnodeEndpoint) + + // Spread + set uniq's the array + userReplicaSet = [...new Set(userReplicaSet)] + } catch (e) { + req.logger.error(`Couldn't get user's replica sets, can't use cnode gateways in saveFileForMultihash`) + } } if (!walletPublicKeys.includes(fetchedWalletPublicKey)) { diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js new file mode 100644 index 00000000000..5e588b6621e --- /dev/null +++ b/creator-node/src/routes/vectorClock.js @@ -0,0 +1,117 @@ +const models = require('../models') +const { sequelize } = require('../models') +const { handleResponse, successResponse, errorResponseServerError } = require('../apiHelpers') +const axios = require('axios') + +module.exports = function (app) { + app.post('/vector_clock_backfill/:wallet', handleResponse(async (req, res, next) => { + const walletPublicKey = req.params.wallet + + const transaction = await models.sequelize.transaction() + try { + // Fetch cnodeUser for each walletPublicKey. + // lock the CNodeUser table so no other tx can write to it while this is in progress + const cnodeUser = await models.CNodeUser.findOne({ + where: { + walletPublicKey: walletPublicKey + }, + transaction, + lock: transaction.LOCK.UPDATE // this makes the query SELECT ... FOR UPDATE + }) + // early exit if clock values have been added for CNodeUser + if (cnodeUser.clock && cnodeUser.clock > 0) return successResponse({ status: 'Already ran successfully!' }) + + // Fetch all data for cnodeUserUUIDs: audiusUsers, tracks, files. + let [audiusUsers, tracks, files] = await Promise.all([ + models.AudiusUser.findAll({ where: { cnodeUserUUID: cnodeUser.cnodeUserUUID }, transaction, raw: true }), + models.Track.findAll({ where: { cnodeUserUUID: cnodeUser.cnodeUserUUID }, transaction, raw: true }), + models.File.findAll({ where: { cnodeUserUUID: cnodeUser.cnodeUserUUID }, transaction, raw: true }) + ]) + + audiusUsers.map(record => record.type = 'AudiusUser') + tracks.map(record => record.type = 'Track') + // if it doesn't have a type it's a file + + let allRecords = audiusUsers.concat(tracks, files) + // sort in chronological order, oldest first + allRecords.sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt)) + + // reset these values + audiusUsers = [] + tracks = [] + files = [] + let clockRecords = [] + + let clock = 0 + allRecords.map(record => { + clock += 1 + let clockRecord = { cnodeUserUUID: cnodeUser.cnodeUserUUID, clock, createdAt: record.createdAt } + if (record.type === 'AudiusUser') { + audiusUsers.push({ ...record, clock }) + clockRecord.sourceTable = 'AudiusUser' + } else if (record.type === 'Track') { + tracks.push({ ...record, clock }) + clockRecord.sourceTable = 'Track' + } else { + files.push({ ...record, clock }) + clockRecord.sourceTable = 'File' + } + clockRecords.push(clockRecord) + }) + console.log('final clock value', clock) + + // delete the existing records + await models.AudiusUser.destroy({ + where: { cnodeUserUUID: cnodeUser.cnodeUserUUID }, + transaction + }) + + await models.Track.destroy({ + where: { cnodeUserUUID: cnodeUser.cnodeUserUUID }, + transaction + }) + + await models.File.destroy({ + where: { cnodeUserUUID: cnodeUser.cnodeUserUUID }, + transaction + }) + + // insert the new records + // chunk files by 10000 records to insert if > 10000 + if (files.length > 10000) { + for (let i = 0; i <= files.length; i += 10000) { + console.log('writing files from idx', i, i + 10000) + await models.File.bulkCreate(files.slice(i, i + 10000), { transaction }) + } + } else { + await models.File.bulkCreate(files, { transaction }) + } + await models.Track.bulkCreate(tracks, { transaction }) + await models.AudiusUser.bulkCreate(audiusUsers, { transaction }) + await models.ClockRecord.bulkCreate(clockRecords, { transaction }) + await cnodeUser.update({ clock }, { transaction }) + + await transaction.commit() + + // trigger secondary syncs here + const axiosReq = { + baseURL: 'http://docker.for.mac.localhost:4000', + url: '/vector_clock_sync', + method: 'post', + data: { + wallet: [walletPublicKey], + creator_node_endpoint: 'http://docker.for.mac.localhost:4000', + immediate: true, + db_only_sync: true + } + } + await axios(axiosReq) + + return successResponse() + } catch (e) { + console.error(e) + await transaction.rollback() + return errorResponseServerError(e.message) + } + })) +} From e9411110e648541a6d8f36596645b1dd02e15c64 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Sun, 20 Sep 2020 21:59:41 +0000 Subject: [PATCH 24/53] WIP dbManager tests --- creator-node/src/clockManager.test.js | 92 ------------ creator-node/src/dbManager.test.js | 206 ++++++++++++++++++++++++++ creator-node/src/models/file.js | 4 + 3 files changed, 210 insertions(+), 92 deletions(-) delete mode 100644 creator-node/src/clockManager.test.js create mode 100644 creator-node/src/dbManager.test.js diff --git a/creator-node/src/clockManager.test.js b/creator-node/src/clockManager.test.js deleted file mode 100644 index f9c95625a92..00000000000 --- a/creator-node/src/clockManager.test.js +++ /dev/null @@ -1,92 +0,0 @@ -const assert = require('assert') -const proxyquire = require('proxyquire') -const _ = require('lodash') - -const models = require('./models') -const { createStarterCNodeUser } = require('../test/lib/dataSeeds') -const { incrementAndFetchCNodeUserClock, fetchCNodeUserClockSubquery } = require('./clockManager') -const utils = require('./utils') - -describe.skip('Test incrementAndFetchCNodeUserClock', () => { - const req = { - logger: { - error: (msg) => console.log(msg) - } - } - - const initialClockVal = 0 - const incrementClockBy = 1 - - let cnodeUser - - /** Create cnodeUser */ - beforeEach(async () => { - const resp = await createStarterCNodeUser() - req.session = { cnodeUserUUID: resp.cnodeUserUUID } - - // Confirm initial clock val in DB - cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) - console.log(cnodeUser) - // assert.strictEqual(cnodeUser.clock, initialClockVal) - }) - - /** Wipe all CNodeUsers + dependent data */ - afterEach(async () => { - await models.CNodeUser.destroy({ - where: {}, - truncate: true, - cascade: true // cascades delete to all rows with foreign key on cnodeUser - }) - }) - - - it('Sequential increment&Fetch', async () => { - // Explicitly pass in incrementBy value - const newClockVal1 = await incrementAndFetchCNodeUserClock(req, incrementClockBy) - - // Confirm function response - assert.strictEqual(newClockVal1, initialClockVal + incrementClockBy) - - // Confirm clock val in DB - cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) - assert.strictEqual(cnodeUser.clock, initialClockVal + incrementClockBy) - - // Use default incrementBy value - const newClockVal2 = await incrementAndFetchCNodeUserClock(req) - - // Confirm function response - assert.strictEqual(newClockVal2, newClockVal1 + incrementClockBy) - - // Confirm clock val in DB - cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: req.session.cnodeUserUUID } }) - assert.strictEqual(cnodeUser.clock, newClockVal1 + incrementClockBy) - }) - - it('Concurrent increment&Fetch', async () => { - // Add global sequelize hook to add timeout before cnodeUser.update calls to force concurrent transactions - const modelsCopy = models - modelsCopy.sequelize.addHook('beforeUpdate', async (instance, options) => { - if (instance.constructor.name === 'CNodeUser') { - await utils.timeout(5000) - } - }) - - // Replace required models instance with modified models instance - proxyquire('./incrementAndFetchCNodeUserClock.js', { '../models': modelsCopy }) - - // Fire 10 increment&Fetch calls in parallel - const arr = _.range(1, 11) // [1,2,3,4,5,6,7,8,9,10] - const returnedClockVals = await Promise.all(arr.map(async (i) => { - console.log(`calling increment and fetch ${i}...`) - return incrementAndFetchCNodeUserClock(req) - })) - - // Ensure returned clock values include no duplicates and include each value from 1-10, in any order - const returnedClockValsSorted = returnedClockVals.sort((a, b) => a - b) - assert.deepStrictEqual(returnedClockValsSorted, arr) - }) - - it('Force blocked requests to fail', async () => { - // return - }) -}) diff --git a/creator-node/src/dbManager.test.js b/creator-node/src/dbManager.test.js new file mode 100644 index 00000000000..f3d184635a0 --- /dev/null +++ b/creator-node/src/dbManager.test.js @@ -0,0 +1,206 @@ +const assert = require('assert') +const proxyquire = require('proxyquire') +const _ = require('lodash') + +const models = require('./models') +const { createStarterCNodeUser } = require('../test/lib/dataSeeds') +const DBManager = require('./dbManager') +const utils = require('./utils') + +describe.only('Test createNewDataRecord()', () => { + const req = { + logger: { + error: (msg) => console.log(msg) + } + } + + const getCNodeUser = async (cnodeUserUUID) => { + return (await models.CNodeUser.findOne({ where: { cnodeUserUUID } })).dataValues + } + + const initialClockVal = 0 + + let cnodeUserUUID + + /** Create cnodeUser + confirm initial clock state */ + beforeEach(async () => { + await models.CNodeUser.destroy({ + where: {}, + truncate: true, + cascade: true // cascades delete to all rows with foreign key on cnodeUser + }) + const resp = await createStarterCNodeUser() + cnodeUserUUID = resp.cnodeUserUUID + req.session = { cnodeUserUUID } + + // Confirm initial clock val in DB + const cnodeUser = await getCNodeUser(cnodeUserUUID) + assert.strictEqual(cnodeUser.clock, initialClockVal) + }) + + /** Wipe all CNodeUsers + dependent data */ + afterEach(async () => { + return + await models.CNodeUser.destroy({ + where: {}, + truncate: true, + cascade: true // cascades delete to all rows with foreign key on cnodeUser + }) + }) + + // it.only('test', () => {}) + + it('Sequential createNewDataRecord - create 2 records', async () => { + const sourceTable = 'File' + + /** + * CREATE RECORD 1 + */ + + // Create new Data record + let transaction = await models.sequelize.transaction() + let createFileQueryObj = { + multihash: 'testMultihash', + sourceFile: 'testSourceFile', + storagePath: 'testStoragePath', + type: 'metadata' // TODO - replace with models enum + } + let createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + await transaction.commit() + + // Validate returned file object + assert.strictEqual(createdFile.cnodeUserUUID, cnodeUserUUID) + assert.strictEqual(createdFile.clock, initialClockVal + 1) + + // Validate CNodeUsers table state + let cnodeUser = await getCNodeUser(cnodeUserUUID) + assert.strictEqual(cnodeUser.clock, initialClockVal + 1) + + // Validate ClockRecords table state + let clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }}) + assert.strictEqual(clockRecords.length, 1) + let clockRecord = clockRecords[0].dataValues + assert.strictEqual(clockRecord.clock, initialClockVal + 1) + assert.strictEqual(clockRecord.sourceTable, sourceTable) + + // Validate Files table state + let files = await models.File.findAll({ where: { cnodeUserUUID }}) + assert.strictEqual(files.length, 1) + let file = files[0].dataValues + assert.strictEqual(file.clock, initialClockVal + 1) + + /** + * CREATE RECORD 2 + */ + + // Create new Data record + transaction = await models.sequelize.transaction() + createFileQueryObj = { + multihash: 'testMultihash2', + sourceFile: 'testSourceFile2', + storagePath: 'testStoragePath2', + type: 'metadata' // TODO - replace with models enum + } + createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + await transaction.commit() + + // Validate returned file object + assert.strictEqual(createdFile.cnodeUserUUID, cnodeUserUUID) + assert.strictEqual(createdFile.clock, initialClockVal + 2) + + // Validate CNodeUsers table state + cnodeUser = await getCNodeUser(cnodeUserUUID) + assert.strictEqual(cnodeUser.clock, initialClockVal + 2) + + // Validate ClockRecords table state + clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['updatedAt', 'DESC']] }) + assert.strictEqual(clockRecords.length, 2) + clockRecord = clockRecords[0].dataValues + assert.strictEqual(clockRecord.clock, initialClockVal + 2) + assert.strictEqual(clockRecord.sourceTable, sourceTable) + + // Validate Files table state + files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['updatedAt', 'DESC']] }) + assert.strictEqual(files.length, 2) + file = files[0].dataValues + assert.strictEqual(file.clock, initialClockVal + 2) + }) + + it.skip('Concurrent createNewDataRecord - create 10 records', async () => { + // test two concurrent in separate transactions (simulates routes) + const t1 = await models.sequelize.transaction() + const t2 = await models.sequelize.transaction() + + // Create new Data record + let createFileQueryObj = { + multihash: 'testMultihash', + sourceFile: 'testSourceFile', + storagePath: 'testStoragePath', + type: 'metadata' // TODO - replace with models enum + } + let createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + await t1.commit() + }) + + it.skip('Concurrent pt2', async () => { + // test concurrent in same transaction -> this should break + }) + + + + it.only('Concurrent createNewDataRecord', async () => { + const sourceTable = 'File' + try { + // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops + const modelsCopy = models + modelsCopy.sequelize.addHook('beforeCreate', async (instance, options) => { + if (instance.constructor.name === 'File') { + await utils.timeout(1000) + } + }) + + // Replace required models instance with modified models instance + proxyquire('./dbManager', { './models': modelsCopy }) + + const transaction = await models.sequelize.transaction() + + // Fire 10 increment&Fetch calls in parallel + const arr = _.range(1, 8) // [1,2,3,4,5,6,7,8,9,10] + const returnedData = await Promise.all(arr.map(async (i) => { + console.log(`calling createNewDataRecord ${i}...`) + + const createFileQueryObj = { + multihash: 'testMultihash2', + sourceFile: 'testSourceFile2', + storagePath: 'testStoragePath2', + type: 'metadata' // TODO - replace with models enum + } + const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + + return createdFile + })) + await transaction.commit() + } catch (e) { + console.log(e) + throw new Error(e) + } + + /** + * TODO + * ensure concurrent with multiple transactions always works + * ensure concurrent in single transaction always fails due to SequelizeUniqueConstraintError + * - test with timeout before ClockRecord to confirm ClockRecord.pkey + * - test with timeout before File to confirm File.pkey + * - test with timeout before + */ + + + // // Ensure returned clock values include no duplicates and include each value from 1-10, in any order + // const returnedClockValsSorted = returnedClockVals.sort((a, b) => a - b) + // assert.deepStrictEqual(returnedClockValsSorted, arr) + }) + + it('Force blocked requests to fail', async () => { + // return + }) +}) diff --git a/creator-node/src/models/file.js b/creator-node/src/models/file.js index 4c7479531d3..fb22035210a 100644 --- a/creator-node/src/models/file.js +++ b/creator-node/src/models/file.js @@ -67,6 +67,10 @@ module.exports = (sequelize, DataTypes) => { }, { fields: ['trackBlockchainId'] + }, + { + unique: true, + fields: ['cnodeUserUUID', 'clock'] } ] }) From 6316dd947480dfa1f2528154a5ed976dc242399d Mon Sep 17 00:00:00 2001 From: SidSethi Date: Mon, 21 Sep 2020 15:23:33 +0000 Subject: [PATCH 25/53] Completed dbManager tests --- creator-node/src/dbManager.test.js | 170 +++++++++++++++++++---------- creator-node/src/models/index.js | 4 - 2 files changed, 112 insertions(+), 62 deletions(-) diff --git a/creator-node/src/dbManager.test.js b/creator-node/src/dbManager.test.js index f3d184635a0..24805e2b3af 100644 --- a/creator-node/src/dbManager.test.js +++ b/creator-node/src/dbManager.test.js @@ -19,6 +19,7 @@ describe.only('Test createNewDataRecord()', () => { } const initialClockVal = 0 + const timeoutMs = 1000 let cnodeUserUUID @@ -40,7 +41,6 @@ describe.only('Test createNewDataRecord()', () => { /** Wipe all CNodeUsers + dependent data */ afterEach(async () => { - return await models.CNodeUser.destroy({ where: {}, truncate: true, @@ -48,8 +48,6 @@ describe.only('Test createNewDataRecord()', () => { }) }) - // it.only('test', () => {}) - it('Sequential createNewDataRecord - create 2 records', async () => { const sourceTable = 'File' @@ -113,94 +111,150 @@ describe.only('Test createNewDataRecord()', () => { assert.strictEqual(cnodeUser.clock, initialClockVal + 2) // Validate ClockRecords table state - clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['updatedAt', 'DESC']] }) + clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) assert.strictEqual(clockRecords.length, 2) clockRecord = clockRecords[0].dataValues - assert.strictEqual(clockRecord.clock, initialClockVal + 2) assert.strictEqual(clockRecord.sourceTable, sourceTable) + assert.strictEqual(clockRecord.clock, initialClockVal + 2) // Validate Files table state - files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['updatedAt', 'DESC']] }) + files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) assert.strictEqual(files.length, 2) file = files[0].dataValues assert.strictEqual(file.clock, initialClockVal + 2) }) - it.skip('Concurrent createNewDataRecord - create 10 records', async () => { - // test two concurrent in separate transactions (simulates routes) - const t1 = await models.sequelize.transaction() - const t2 = await models.sequelize.transaction() + it('Concurrent createNewDataRecord - successfully makes concurrent calls in separate transactions', async () => { + const sourceTable = 'File' + const numEntries = 5 + + // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops + const modelsCopy = models + modelsCopy.sequelize.addHook('beforeCreate', async (instance, options) => { + if (instance.constructor.name === 'ClockRecord') { + await utils.timeout(timeoutMs) + } + }) - // Create new Data record - let createFileQueryObj = { - multihash: 'testMultihash', - sourceFile: 'testSourceFile', - storagePath: 'testStoragePath', - type: 'metadata' // TODO - replace with models enum - } - let createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) - await t1.commit() - }) + // Replace required models instance with modified models instance + proxyquire('./dbManager', { './models': modelsCopy }) - it.skip('Concurrent pt2', async () => { - // test concurrent in same transaction -> this should break - }) + // Make multiple concurrent calls - create a transaction for each call + const arr = _.range(1, numEntries + 1) // [1, 2, ..., numEntries] + let createdFiles = await Promise.all(arr.map(async (i) => { + const transaction = await models.sequelize.transaction() + const createFileQueryObj = { + multihash: 'testMultihash', + sourceFile: 'testSourceFile', + storagePath: 'testStoragePath', + type: 'metadata' // TODO - replace with models enum + } + const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + await transaction.commit() + return createdFile + })) + // Validate returned file objects + createdFiles = _.orderBy(createdFiles, ['createdAt'], ['asc']) + createdFiles.forEach((createdFile, index) => { + assert.strictEqual(createdFile.cnodeUserUUID, cnodeUserUUID) + assert.strictEqual(createdFile.clock, initialClockVal + 1 + index) + }) - it.only('Concurrent createNewDataRecord', async () => { - const sourceTable = 'File' - try { - // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops - const modelsCopy = models - modelsCopy.sequelize.addHook('beforeCreate', async (instance, options) => { - if (instance.constructor.name === 'File') { - await utils.timeout(1000) - } - }) + // Validate CNodeUsers table state + const cnodeUser = await getCNodeUser(cnodeUserUUID) + assert.strictEqual(cnodeUser.clock, initialClockVal + numEntries) - // Replace required models instance with modified models instance - proxyquire('./dbManager', { './models': modelsCopy }) + // Validate ClockRecords table state + const clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'ASC']] }) + assert.strictEqual(clockRecords.length, numEntries) + clockRecords.forEach((clockRecord, index) => { + clockRecord = clockRecord.dataValues + assert.strictEqual(clockRecord.sourceTable, sourceTable) + assert.strictEqual(clockRecord.clock, initialClockVal + 1 + index) + }) - const transaction = await models.sequelize.transaction() + // Validate Files table state + const files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'ASC']] }) + assert.strictEqual(files.length, numEntries) + files.forEach((file, index) => { + file = file.dataValues + assert.strictEqual(file.clock, initialClockVal + 1 + index) + }) + }) - // Fire 10 increment&Fetch calls in parallel - const arr = _.range(1, 8) // [1,2,3,4,5,6,7,8,9,10] - const returnedData = await Promise.all(arr.map(async (i) => { - console.log(`calling createNewDataRecord ${i}...`) + it('Concurrent createNewDataRecord - fails to make concurrent calls in a single transaction due to ClockRecords_pkey', async () => { + const sourceTable = 'File' + const numEntries = 5 + + // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops + const modelsCopy = models + modelsCopy.sequelize.addHook('beforeCreate', async (instance, options) => { + if (instance.constructor.name === 'ClockRecord') { + await utils.timeout(timeoutMs) + } + }) + + // Replace required models instance with modified models instance + proxyquire('./dbManager', { './models': modelsCopy }) + // Attempt to make multiple concurrent calls, re-using the same transaction each time + const transaction = await models.sequelize.transaction() + try { + const arr = _.range(1, numEntries + 1) // [1, 2, ..., numEntries] + await Promise.all(arr.map(async (i) => { const createFileQueryObj = { - multihash: 'testMultihash2', - sourceFile: 'testSourceFile2', - storagePath: 'testStoragePath2', + multihash: 'testMultihash', + sourceFile: 'testSourceFile', + storagePath: 'testStoragePath', type: 'metadata' // TODO - replace with models enum } const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) - return createdFile })) await transaction.commit() } catch (e) { - console.log(e) - throw new Error(e) + await transaction.rollback() + assert.strictEqual(e.name, 'SequelizeUniqueConstraintError') + assert.strictEqual(e.original.message, 'duplicate key value violates unique constraint "ClockRecords_pkey"') } - + /** - * TODO - * ensure concurrent with multiple transactions always works - * ensure concurrent in single transaction always fails due to SequelizeUniqueConstraintError - * - test with timeout before ClockRecord to confirm ClockRecord.pkey - * - test with timeout before File to confirm File.pkey - * - test with timeout before + * Confirm none of the rows were written to DB */ + // Validate CNodeUsers table state + cnodeUser = await getCNodeUser(cnodeUserUUID) + assert.strictEqual(cnodeUser.clock, initialClockVal) + + // Validate ClockRecords table state + clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) + assert.strictEqual(clockRecords.length, 0) - // // Ensure returned clock values include no duplicates and include each value from 1-10, in any order - // const returnedClockValsSorted = returnedClockVals.sort((a, b) => a - b) - // assert.deepStrictEqual(returnedClockValsSorted, arr) + // Validate Files table state + files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) + assert.strictEqual(files.length, 0) }) - it('Force blocked requests to fail', async () => { - // return + it('Confirm file.pkey will block duplicate clock vals from being written', async () => { + const transaction = await models.sequelize.transaction() + try { + const createFileQueryObj = { + cnodeUserUUID, + multihash: 'testMultihash', + sourceFile: 'testSourceFile', + storagePath: 'testStoragePath', + type: 'metadata', // TODO - replace with models enum + clock: 0 + } + await models.File.create(createFileQueryObj, { transaction }) + await models.File.create(createFileQueryObj, { transaction }) + await transaction.commit() + } catch (e) { + await transaction.rollback() + assert.strictEqual(e.name, 'SequelizeUniqueConstraintError') + assert.strictEqual(e.original.message, 'duplicate key value violates unique constraint "Files_unique_(cnodeUserUUID,clock)"') + } }) }) diff --git a/creator-node/src/models/index.js b/creator-node/src/models/index.js index de1c83b66e0..b7b3c95d569 100644 --- a/creator-node/src/models/index.js +++ b/creator-node/src/models/index.js @@ -12,10 +12,6 @@ const db = {} const sequelize = new Sequelize(globalConfig.get('dbUrl'), { logging: true, operatorsAliases: false, - // dialectOptions: { - // idleTimeoutMillis: 500, - // connectionTimeoutMillis: 500 - // }, pool: { max: 100, min: 5, From 7c86257d4332f3f6941f54de8196f3434164c6df Mon Sep 17 00:00:00 2001 From: SidSethi Date: Mon, 21 Sep 2020 17:30:56 +0000 Subject: [PATCH 26/53] Add indexes to sequelize models --- creator-node/src/dbManager.test.js | 2 +- creator-node/src/models/audiususer.js | 9 ++++++++- creator-node/src/models/track.js | 9 ++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/creator-node/src/dbManager.test.js b/creator-node/src/dbManager.test.js index 24805e2b3af..47b6621397e 100644 --- a/creator-node/src/dbManager.test.js +++ b/creator-node/src/dbManager.test.js @@ -40,7 +40,7 @@ describe.only('Test createNewDataRecord()', () => { }) /** Wipe all CNodeUsers + dependent data */ - afterEach(async () => { + after(async () => { await models.CNodeUser.destroy({ where: {}, truncate: true, diff --git a/creator-node/src/models/audiususer.js b/creator-node/src/models/audiususer.js index 49685ae75c0..839322e350c 100644 --- a/creator-node/src/models/audiususer.js +++ b/creator-node/src/models/audiususer.js @@ -31,7 +31,14 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.UUID, allowNull: true } - }, {}) + }, { + indexes: [ + { + unique: true, + fields: ['blockchainId', 'clock'] + } + ] + }) AudiusUser.associate = function (models) { AudiusUser.belongsTo(models.CNodeUser, { foreignKey: 'cnodeUserUUID', diff --git a/creator-node/src/models/track.js b/creator-node/src/models/track.js index acd9af8cdf2..3f47185c5db 100644 --- a/creator-node/src/models/track.js +++ b/creator-node/src/models/track.js @@ -28,7 +28,14 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.UUID, allowNull: true } - }, {}) + }, { + indexes: [ + { + unique: true, + fields: ['blockchainId', 'clock'] + } + ] + }) Track.associate = function (models) { Track.belongsTo(models.CNodeUser, { From e02a5949a1dc110213c8fa007215823f637d43d4 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Mon, 21 Sep 2020 23:24:00 +0000 Subject: [PATCH 27/53] Change createNewDataRecord modelsInstance abstraction + add ClockRecord enum integration test --- .../migrations/20200918150546-add-clock.js | 1 - creator-node/src/dbManager.js | 45 ++--- creator-node/src/models/clockRecord.js | 1 - creator-node/src/routes/audiusUsers.js | 4 +- creator-node/src/routes/files.js | 4 +- creator-node/src/routes/tracks.js | 8 +- creator-node/{src => test}/dbManager.test.js | 172 +++++++++++++----- 7 files changed, 150 insertions(+), 85 deletions(-) rename creator-node/{src => test}/dbManager.test.js (62%) diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index f933e72b522..eb1829d1387 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -145,7 +145,6 @@ async function createClockRecordsTable (queryInterface, Sequelize, transaction) allowNull: false }, sourceTable: { - // TODO - if this doesn't work, use models/file.js:L46 type: Sequelize.ENUM('AudiusUser', 'Track', 'File'), allowNull: false }, diff --git a/creator-node/src/dbManager.js b/creator-node/src/dbManager.js index 893fd1afcb3..1d91a0cb514 100644 --- a/creator-node/src/dbManager.js +++ b/creator-node/src/dbManager.js @@ -2,14 +2,8 @@ const models = require('./models') const sequelize = models.sequelize /** -TODO - consider moving to inside /src/models - -TODO consider adding hook to ensure write op can never set clockVal to anything <= current - -TODO - any further constraint enforcement needed? -- DataTables all FK to Clocks table - - Clocks table has unique constraint -// - dataTables have unique C on (userId + clock) + * TODO - add DataTables all composite FK to Clocks table + * - https://stackoverflow.com/questions/9984022/postgres-fk-referencing-composite-pk */ class DBManager { @@ -19,18 +13,19 @@ class DBManager { * Steps: * 1. increments cnodeUser clock value * 2. insert new ClockRecord entry with new clock value - * 3. insert new File entry with queryObj and new clock value + * 3. insert new Data Table (File, Track, AudiusUser) entry with queryObj and new clock value * * After initial increment, clock values are read as subquery without reading into JS to guarantee atomicity - * - * TODO - flesh out jsdoc - * @param {*} queryObj - object with file insert data - * @param {*} cnodeUserUUID - * @param {*} transaction - * + * + * * TODO - flesh out jsdoc + * @param {*} queryObj + * @param {*} cnodeUserUUID + * @param {*} sequelizeTableInstance + * @param {*} transaction + * * TODO - returns */ - static async createNewDataRecord (queryObj, cnodeUserUUID, sourceTable, transaction) { + static async createNewDataRecord (queryObj, cnodeUserUUID, sequelizeTableInstance, transaction) { // Increment CNodeUser.clock value by 1 await models.CNodeUser.increment('clock', { where: { cnodeUserUUID }, @@ -44,34 +39,24 @@ class DBManager { await models.ClockRecord.create({ cnodeUserUUID, clock: selectCNodeUserClockSubqueryLiteral, - sourceTable + sourceTable: sequelizeTableInstance.name }, { transaction }) // Add cnodeUserUUID + clock value to queryObj queryObj.cnodeUserUUID = cnodeUserUUID queryObj.clock = selectCNodeUserClockSubqueryLiteral - // Create new File entry with queryObj - const modelsInstance = _getModelsInstance(sourceTable) - const file = await modelsInstance.create(queryObj, { transaction }) + // Create new Data table entry with queryObj + const file = await sequelizeTableInstance.create(queryObj, { transaction }) return file.dataValues } } -const _SourceTableToModelInstanceMap = { - 'AudiusUser': models.AudiusUser, - 'Track': models.Track, - 'File': models.File -} - -function _getModelsInstance (sourceTable) { - return _SourceTableToModelInstanceMap[sourceTable] -} - /** * @dev source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause * @param {*} cnodeUserUUID + * return string "select * from clock where cnodeuseruuid" */ function _getSelectCNodeUserClockSubqueryLiteral (cnodeUserUUID) { const subquery = sequelize.dialect.QueryGenerator.selectQuery('CNodeUsers', { diff --git a/creator-node/src/models/clockRecord.js b/creator-node/src/models/clockRecord.js index b0772a97f75..8106578f1fb 100644 --- a/creator-node/src/models/clockRecord.js +++ b/creator-node/src/models/clockRecord.js @@ -27,7 +27,6 @@ module.exports = (sequelize, DataTypes) => { allowNull: false }, sourceTable: { - // TODO - if this doesn't work, use models/file.js:L46 type: DataTypes.ENUM( ...Object.values(SourceTableTypesObj) ), diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index 494c627b3b2..4b714b3450e 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -39,7 +39,7 @@ module.exports = function (app) { storagePath: dstPath, type: 'metadata' // TODO - replace with models enum } - const file = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, 'File', transaction) + const file = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, models.File, transaction) fileUUID = file.fileUUID await transaction.commit() @@ -102,7 +102,7 @@ module.exports = function (app) { coverArtFileUUID, profilePicFileUUID } - await DBManager.createNewDataRecord(createAudiusUserQueryObj, cnodeUserUUID, 'AudiusUser', transaction) + await DBManager.createNewDataRecord(createAudiusUserQueryObj, cnodeUserUUID, models.AudiusUser, transaction) // Update cnodeUser.latestBlockNumber // TODO - can this be deprecated with new clock logic? diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js index 43f0d924625..3cf45f55580 100644 --- a/creator-node/src/routes/files.js +++ b/creator-node/src/routes/files.js @@ -296,7 +296,7 @@ module.exports = function (app) { storagePath: resizeResp.dir.dirDestPath, type: 'dir' // TODO - replace with models enum } - await DBManager.createNewDataRecord(createDirFileQueryObj, cnodeUserUUID, 'File', transaction) + await DBManager.createNewDataRecord(createDirFileQueryObj, cnodeUserUUID, models.File, transaction) // Record all image res file entries in DB // Must be written sequentially to ensure clock values are correctly incremented and populated @@ -309,7 +309,7 @@ module.exports = function (app) { dirMultihash: resizeResp.dir.dirCID, fileName: file.sourceFile.split('/').slice(-1)[0] } - await DBManager.createNewDataRecord(createImageFileQueryObj, cnodeUserUUID, 'File', transaction) + await DBManager.createNewDataRecord(createImageFileQueryObj, cnodeUserUUID, models.File, transaction) } req.logger.info(`route time = ${Date.now() - routestart}`) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index e412b1b57c1..1d16ee569a3 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -124,7 +124,7 @@ module.exports = function (app) { storagePath: transcodeFileIPFSResp.dstPath, type: 'copy320' // TODO - replace with models enum } - const file = await DBManager.createNewDataRecord(createTranscodeFileQueryObj, cnodeUserUUID, 'File', transaction) + const file = await DBManager.createNewDataRecord(createTranscodeFileQueryObj, cnodeUserUUID, models.File, transaction) transcodeFileUUID = file.fileUUID // Record all segment file entries in DB @@ -136,7 +136,7 @@ module.exports = function (app) { storagePath: dstPath, type: 'track' // TODO - replace with models enum } - await DBManager.createNewDataRecord(createSegmentFileQueryObj, cnodeUserUUID, 'File', transaction) + await DBManager.createNewDataRecord(createSegmentFileQueryObj, cnodeUserUUID, models.File, transaction) } await transaction.commit() @@ -248,7 +248,7 @@ module.exports = function (app) { storagePath: dstPath, type: 'metadata' // TODO - replace with models enum } - const file = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, 'File', transaction) + const file = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, models.File, transaction) fileUUID = file.fileUUID await transaction.commit() @@ -330,7 +330,7 @@ module.exports = function (app) { blockchainId: blockchainTrackId, coverArtFileUUID } - const track = await DBManager.createNewDataRecord(createTrackQueryObj, cnodeUserUUID, 'Track', transaction) + const track = await DBManager.createNewDataRecord(createTrackQueryObj, cnodeUserUUID, models.Track, transaction) /** * Associate matching transcode & segment files on DB with new/updated track diff --git a/creator-node/src/dbManager.test.js b/creator-node/test/dbManager.test.js similarity index 62% rename from creator-node/src/dbManager.test.js rename to creator-node/test/dbManager.test.js index 47b6621397e..57ece7fd484 100644 --- a/creator-node/src/dbManager.test.js +++ b/creator-node/test/dbManager.test.js @@ -2,12 +2,12 @@ const assert = require('assert') const proxyquire = require('proxyquire') const _ = require('lodash') -const models = require('./models') -const { createStarterCNodeUser } = require('../test/lib/dataSeeds') -const DBManager = require('./dbManager') -const utils = require('./utils') +const models = require('../src/models') +const { createStarterCNodeUser } = require('./lib/dataSeeds') +const DBManager = require('../src/dbManager') +const utils = require('../src/utils') -describe.only('Test createNewDataRecord()', () => { +describe('Test createNewDataRecord()', () => { const req = { logger: { error: (msg) => console.log(msg) @@ -21,15 +21,10 @@ describe.only('Test createNewDataRecord()', () => { const initialClockVal = 0 const timeoutMs = 1000 - let cnodeUserUUID + let cnodeUserUUID, createFileQueryObj - /** Create cnodeUser + confirm initial clock state */ + /** Create cnodeUser + confirm initial clock state + define global vars */ beforeEach(async () => { - await models.CNodeUser.destroy({ - where: {}, - truncate: true, - cascade: true // cascades delete to all rows with foreign key on cnodeUser - }) const resp = await createStarterCNodeUser() cnodeUserUUID = resp.cnodeUserUUID req.session = { cnodeUserUUID } @@ -37,6 +32,13 @@ describe.only('Test createNewDataRecord()', () => { // Confirm initial clock val in DB const cnodeUser = await getCNodeUser(cnodeUserUUID) assert.strictEqual(cnodeUser.clock, initialClockVal) + + createFileQueryObj = { + multihash: 'testMultihash', + sourceFile: 'testSourceFile', + storagePath: 'testStoragePath', + type: 'metadata' // TODO - replace with models enum + } }) /** Wipe all CNodeUsers + dependent data */ @@ -49,7 +51,7 @@ describe.only('Test createNewDataRecord()', () => { }) it('Sequential createNewDataRecord - create 2 records', async () => { - const sourceTable = 'File' + const sequelizeTableInstance = models.File /** * CREATE RECORD 1 @@ -57,13 +59,7 @@ describe.only('Test createNewDataRecord()', () => { // Create new Data record let transaction = await models.sequelize.transaction() - let createFileQueryObj = { - multihash: 'testMultihash', - sourceFile: 'testSourceFile', - storagePath: 'testStoragePath', - type: 'metadata' // TODO - replace with models enum - } - let createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + let createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sequelizeTableInstance, transaction) await transaction.commit() // Validate returned file object @@ -79,7 +75,7 @@ describe.only('Test createNewDataRecord()', () => { assert.strictEqual(clockRecords.length, 1) let clockRecord = clockRecords[0].dataValues assert.strictEqual(clockRecord.clock, initialClockVal + 1) - assert.strictEqual(clockRecord.sourceTable, sourceTable) + assert.strictEqual(clockRecord.sourceTable, sequelizeTableInstance.name) // Validate Files table state let files = await models.File.findAll({ where: { cnodeUserUUID }}) @@ -99,7 +95,7 @@ describe.only('Test createNewDataRecord()', () => { storagePath: 'testStoragePath2', type: 'metadata' // TODO - replace with models enum } - createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sequelizeTableInstance, transaction) await transaction.commit() // Validate returned file object @@ -114,7 +110,7 @@ describe.only('Test createNewDataRecord()', () => { clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) assert.strictEqual(clockRecords.length, 2) clockRecord = clockRecords[0].dataValues - assert.strictEqual(clockRecord.sourceTable, sourceTable) + assert.strictEqual(clockRecord.sourceTable, sequelizeTableInstance.name) assert.strictEqual(clockRecord.clock, initialClockVal + 2) // Validate Files table state @@ -125,7 +121,7 @@ describe.only('Test createNewDataRecord()', () => { }) it('Concurrent createNewDataRecord - successfully makes concurrent calls in separate transactions', async () => { - const sourceTable = 'File' + const sequelizeTableInstance = models.File const numEntries = 5 // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops @@ -137,19 +133,13 @@ describe.only('Test createNewDataRecord()', () => { }) // Replace required models instance with modified models instance - proxyquire('./dbManager', { './models': modelsCopy }) + proxyquire('../src/dbManager', { './models': modelsCopy }) // Make multiple concurrent calls - create a transaction for each call const arr = _.range(1, numEntries + 1) // [1, 2, ..., numEntries] - let createdFiles = await Promise.all(arr.map(async (i) => { + let createdFiles = await Promise.all(arr.map(async () => { const transaction = await models.sequelize.transaction() - const createFileQueryObj = { - multihash: 'testMultihash', - sourceFile: 'testSourceFile', - storagePath: 'testStoragePath', - type: 'metadata' // TODO - replace with models enum - } - const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sequelizeTableInstance, transaction) await transaction.commit() return createdFile @@ -171,7 +161,7 @@ describe.only('Test createNewDataRecord()', () => { assert.strictEqual(clockRecords.length, numEntries) clockRecords.forEach((clockRecord, index) => { clockRecord = clockRecord.dataValues - assert.strictEqual(clockRecord.sourceTable, sourceTable) + assert.strictEqual(clockRecord.sourceTable, sequelizeTableInstance.name) assert.strictEqual(clockRecord.clock, initialClockVal + 1 + index) }) @@ -185,7 +175,7 @@ describe.only('Test createNewDataRecord()', () => { }) it('Concurrent createNewDataRecord - fails to make concurrent calls in a single transaction due to ClockRecords_pkey', async () => { - const sourceTable = 'File' + const sequelizeTableInstance = models.File const numEntries = 5 // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops @@ -197,20 +187,14 @@ describe.only('Test createNewDataRecord()', () => { }) // Replace required models instance with modified models instance - proxyquire('./dbManager', { './models': modelsCopy }) + proxyquire('../src/dbManager', { './models': modelsCopy }) // Attempt to make multiple concurrent calls, re-using the same transaction each time const transaction = await models.sequelize.transaction() try { const arr = _.range(1, numEntries + 1) // [1, 2, ..., numEntries] - await Promise.all(arr.map(async (i) => { - const createFileQueryObj = { - multihash: 'testMultihash', - sourceFile: 'testSourceFile', - storagePath: 'testStoragePath', - type: 'metadata' // TODO - replace with models enum - } - const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sourceTable, transaction) + await Promise.all(arr.map(async () => { + const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sequelizeTableInstance, transaction) return createdFile })) await transaction.commit() @@ -237,10 +221,56 @@ describe.only('Test createNewDataRecord()', () => { assert.strictEqual(files.length, 0) }) + /** + * Simulates /image_upload and /track_content routes, which write multiple files sequentially in atomic tx + */ + it('Sequential createNewDataRecord - successfully makes multiple sequential calls in single transaction', async () => { + const sequelizeTableInstance = models.File + const numEntries = 5 + + // Make multiple squential calls, re-using the same transaction each time + const transaction = await models.sequelize.transaction() + const arr = _.range(1, numEntries + 1) // [1, 2, ..., numEntries] + const createdFilesResp = [] + for await (const i of arr) { + const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sequelizeTableInstance, transaction) + createdFilesResp.push(createdFile) + } + await transaction.commit() + + // Validate returned file objects + const createdFiles = _.orderBy(createdFilesResp, ['createdAt'], ['asc']) + createdFiles.forEach((createdFile, index) => { + assert.strictEqual(createdFile.cnodeUserUUID, cnodeUserUUID) + assert.strictEqual(createdFile.clock, initialClockVal + 1 + index) + }) + + // Validate CNodeUsers table state + const cnodeUser = await getCNodeUser(cnodeUserUUID) + assert.strictEqual(cnodeUser.clock, initialClockVal + numEntries) + + // Validate ClockRecords table state + const clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'ASC']] }) + assert.strictEqual(clockRecords.length, numEntries) + clockRecords.forEach((clockRecord, index) => { + clockRecord = clockRecord.dataValues + assert.strictEqual(clockRecord.sourceTable, sequelizeTableInstance.name) + assert.strictEqual(clockRecord.clock, initialClockVal + 1 + index) + }) + + // Validate Files table state + const files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'ASC']] }) + assert.strictEqual(files.length, numEntries) + files.forEach((file, index) => { + file = file.dataValues + assert.strictEqual(file.clock, initialClockVal + 1 + index) + }) + }) + it('Confirm file.pkey will block duplicate clock vals from being written', async () => { const transaction = await models.sequelize.transaction() try { - const createFileQueryObj = { + createFileQueryObj = { cnodeUserUUID, multihash: 'testMultihash', sourceFile: 'testSourceFile', @@ -258,3 +288,55 @@ describe.only('Test createNewDataRecord()', () => { } }) }) + +describe('Test ClockRecord model', () => { + it('Confirm only valid sourceTable value can be written to ClockRecords table', async () => { + await models.CNodeUser.destroy({ + where: {}, + truncate: true, + cascade: true // cascades delete to all rows with foreign key on cnodeUser + }) + const cnodeUserUUID = (await createStarterCNodeUser()).cnodeUserUUID + + const validSourceTable = 'AudiusUser' + const invalidSourceTable = 'invalidSourceTable' + + // Confirm ClockRecords insert with validSourceTable value will succeed + await models.ClockRecord.create({ + cnodeUserUUID, + clock: 0, + sourceTable: validSourceTable + }) + + // Confirm ClockRecord was created + const clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID } }) + assert.strictEqual(clockRecords.length, 1) + const clockRecord = clockRecords[0] + assert.strictEqual(clockRecord.cnodeUserUUID, cnodeUserUUID) + assert.strictEqual(clockRecord.clock, 0) + assert.strictEqual(clockRecord.sourceTable, validSourceTable) + + // Confirm ClockRecords insert with invalidSourceTable value will fail due to DB error + try { + await models.sequelize.query(` + INSERT INTO "ClockRecords" + ("cnodeUserUUID","clock","sourceTable","createdAt","updatedAt") + VALUES ( + 'f13d776e-c4a6-4007-93bc-7e625c862873', + 0, + :invalidSourceTable, + '2020-09-21 23:04:06.339 +00:00', + '2020-09-21 23:04:06.339 +00:00' + );`, + { + replacements: { invalidSourceTable }, + type: 'RAW', + raw: true + } + ) + } catch (e) { + assert.strictEqual(e.name, 'SequelizeDatabaseError') + assert.strictEqual(e.original.message, `invalid input value for enum "enum_ClockRecords_sourceTable": "${invalidSourceTable}"`) + } + }) +}) \ No newline at end of file From bf09dac84f483e1afdf8d3b826ae70dc103a957e Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Mon, 21 Sep 2020 19:31:11 -0400 Subject: [PATCH 28/53] simplify export a bit --- creator-node/src/routes/nodeSync.js | 96 ++++++++++++++++++-------- creator-node/src/routes/vectorClock.js | 46 +++++++----- 2 files changed, 95 insertions(+), 47 deletions(-) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 97b714b197b..16eb683cd9e 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -24,21 +24,64 @@ module.exports = function (app) { * } */ app.get('/export', handleResponse(async (req, res) => { + // TODO - allow for offsets in the /export + const limit = 25000 const walletPublicKeys = req.query.wallet_public_key // array const dbOnlySync = (req.query.db_only_sync === true || req.query.db_only_sync === 'true') const transaction = await models.sequelize.transaction() try { // Fetch cnodeUser for each walletPublicKey. - const cnodeUsers = await models.CNodeUser.findAll({ where: { walletPublicKey: walletPublicKeys }, transaction }) + const cnodeUsers = await models.CNodeUser.findAll({ where: { walletPublicKey: walletPublicKeys }, transaction, raw: true }) const cnodeUserUUIDs = cnodeUsers.map((cnodeUser) => cnodeUser.cnodeUserUUID) + const cnodeUserUUID = cnodeUserUUIDs[0] // assume we only have a single wallet being exported for now // Fetch all data for cnodeUserUUIDs: audiusUsers, tracks, files, clockRecords. const [audiusUsers, tracks, files, clockRecords] = await Promise.all([ - models.AudiusUser.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }), - models.Track.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }), - models.File.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }), - models.ClockRecord.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs }, transaction }) + models.AudiusUser.findAll({ + where: { + cnodeUserUUID: cnodeUserUUIDs, + clock: { + [models.Sequelize.Op.lte]: limit + } + }, + order: [['clock', 'ASC']], + transaction, + raw: true + }), + models.Track.findAll({ + where: { + cnodeUserUUID: cnodeUserUUIDs, + clock: { + [models.Sequelize.Op.lte]: limit + } + }, + order: [['clock', 'ASC']], + transaction, + raw: true + }), + models.File.findAll({ + where: { + cnodeUserUUID: cnodeUserUUIDs, + clock: { + [models.Sequelize.Op.lte]: limit + } + }, + order: [['clock', 'ASC']], + transaction, + raw: true + }), + models.ClockRecord.findAll({ + where: { + cnodeUserUUID: cnodeUserUUIDs, + clock: { + [models.Sequelize.Op.lte]: limit + } + }, + order: [['clock', 'ASC']], + transaction, + raw: true + }) ]) await transaction.commit() @@ -46,34 +89,27 @@ module.exports = function (app) { const cnodeUsersDict = {} cnodeUsers.forEach(cnodeUser => { - // Convert sequelize object to plain js object to allow adding additional fields. - const cnodeUserDictObj = cnodeUser.toJSON() - // Add cnodeUserUUID data fields. - cnodeUserDictObj['audiusUsers'] = [] - cnodeUserDictObj['tracks'] = [] - cnodeUserDictObj['files'] = [] - cnodeUserDictObj['clockRecords'] = [] - - cnodeUsersDict[cnodeUser.cnodeUserUUID] = cnodeUserDictObj + cnodeUser['audiusUsers'] = [] + cnodeUser['tracks'] = [] + cnodeUser['files'] = [] + cnodeUser['clockRecords'] = [] + + cnodeUsersDict[cnodeUser.cnodeUserUUID] = cnodeUser + + // TODO - remove this once we no longer have a limit in export + // this just overrides the clock value to the max clock we're sending over to the secondary so it knows + // there's more data to pull + if (cnodeUser.clock > limit) { + console.log("nodeSync.js#export - cnode user clock value is higher than limit, resetting", clockRecords[clockRecords.length-1].clock) + cnodeUser.clock = clockRecords[clockRecords.length-1].clock + } }) - audiusUsers.forEach(audiusUser => { - const audiusUserDictObj = audiusUser.toJSON() - cnodeUsersDict[audiusUserDictObj['cnodeUserUUID']]['audiusUsers'].push(audiusUserDictObj) - }) - tracks.forEach(track => { - let trackDictObj = track.toJSON() - cnodeUsersDict[trackDictObj['cnodeUserUUID']]['tracks'].push(trackDictObj) - }) - files.forEach(file => { - let fileDictObj = file.toJSON() - cnodeUsersDict[fileDictObj['cnodeUserUUID']]['files'].push(fileDictObj) - }) - clockRecords.forEach(clockRecord => { - let clockRecordDictObj = clockRecord.toJSON() - cnodeUsersDict[clockRecordDictObj['cnodeUserUUID']]['clockRecords'].push(clockRecordDictObj) - }) + cnodeUsersDict[cnodeUserUUID]['audiusUsers'] = audiusUsers + cnodeUsersDict[cnodeUserUUID]['tracks'] = tracks + cnodeUsersDict[cnodeUserUUID]['files'] = files + cnodeUsersDict[cnodeUserUUID]['clockRecords'] = clockRecords // Expose ipfs node's peer ID. const ipfs = req.app.get('ipfsAPI') diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 5e588b6621e..2bf55773393 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -86,32 +86,44 @@ module.exports = function (app) { } else { await models.File.bulkCreate(files, { transaction }) } + await models.Track.bulkCreate(tracks, { transaction }) + await models.AudiusUser.bulkCreate(audiusUsers, { transaction }) - await models.ClockRecord.bulkCreate(clockRecords, { transaction }) - await cnodeUser.update({ clock }, { transaction }) - - await transaction.commit() - - // trigger secondary syncs here - const axiosReq = { - baseURL: 'http://docker.for.mac.localhost:4000', - url: '/vector_clock_sync', - method: 'post', - data: { - wallet: [walletPublicKey], - creator_node_endpoint: 'http://docker.for.mac.localhost:4000', - immediate: true, - db_only_sync: true + + if (clockRecords.length > 10000) { + for (let i = 0; i <= clockRecords.length; i += 10000) { + console.log('writing clockrecords from idx', i, i + 10000) + await models.ClockRecord.bulkCreate(clockRecords.slice(i, i + 10000), { transaction }) } + } else { + await models.ClockRecord.bulkCreate(clockRecords, { transaction }) } - await axios(axiosReq) + + await cnodeUser.update({ clock }, { transaction }) - return successResponse() + await transaction.commit() } catch (e) { console.error(e) await transaction.rollback() return errorResponseServerError(e.message) } + + // trigger secondary syncs here + const axiosReq = { + baseURL: 'http://docker.for.mac.localhost:4000', + url: '/vector_clock_sync', + method: 'post', + data: { + wallet: [walletPublicKey], + creator_node_endpoint: 'http://docker.for.mac.localhost:4000', + immediate: true, + db_only_sync: true + } + } + await axios(axiosReq) + + return successResponse() + })) } From 3066e1b0b189a68d2553eba028213859c06fd8fc Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 10:13:01 -0400 Subject: [PATCH 29/53] make the call to sync in the vector_clock_backfill route? --- creator-node/src/routes/vectorClock.js | 32 +++++++++++++++++--------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 2bf55773393..afe246e0634 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -6,6 +6,7 @@ const axios = require('axios') module.exports = function (app) { app.post('/vector_clock_backfill/:wallet', handleResponse(async (req, res, next) => { const walletPublicKey = req.params.wallet + const { primary, secondaries } = req.body const transaction = await models.sequelize.transaction() try { @@ -18,6 +19,10 @@ module.exports = function (app) { transaction, lock: transaction.LOCK.UPDATE // this makes the query SELECT ... FOR UPDATE }) + + // early exit if cnodeUser not found on primary + if (!cnodeUser) return successResponse('No cnodeUser record found on the primary') + // early exit if clock values have been added for CNodeUser if (cnodeUser.clock && cnodeUser.clock > 0) return successResponse({ status: 'Already ran successfully!' }) @@ -110,18 +115,23 @@ module.exports = function (app) { } // trigger secondary syncs here - const axiosReq = { - baseURL: 'http://docker.for.mac.localhost:4000', - url: '/vector_clock_sync', - method: 'post', - data: { - wallet: [walletPublicKey], - creator_node_endpoint: 'http://docker.for.mac.localhost:4000', - immediate: true, - db_only_sync: true - } + if (secondaries && secondaries.length > 0) { + await Promise.all(secondaries.map(secondary => { + console.log('calling sync to secondary', secondary) + const axiosReq = { + baseURL: secondary, + url: '/vector_clock_sync', + method: 'post', + data: { + wallet: [walletPublicKey], + creator_node_endpoint: 'http://docker.for.mac.localhost:4000', + immediate: true, + db_only_sync: true + } + } + return axios(axiosReq) + })) } - await axios(axiosReq) return successResponse() From b92d1391ccd3898516c5ae84f0b99a7e883d4250 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 15:32:06 -0400 Subject: [PATCH 30/53] migrations and bug fix --- .../20200911004845-allow-track-and-audiusUsers-appends.js | 7 ------- .../sequelize/migrations/20200918150546-add-clock.js | 4 ---- creator-node/src/routes/vectorClock.js | 2 +- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js index 4874026f612..d7329df0198 100644 --- a/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js +++ b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js @@ -36,13 +36,6 @@ module.exports = { -- No fkey from Files to Tracks because we don't have a unique constraint on trackUUID or blockchainId on Tracks so postgres would reject the fkey ALTER TABLE "Files" ADD CONSTRAINT "Files_cnodeUserUUID_fkey" FOREIGN KEY ("cnodeUserUUID") REFERENCES "CNodeUsers" ("cnodeUserUUID") ON DELETE RESTRICT; - -- 5 foreign key constraints get dropped in the CASCADE, so add them back in for the new table - ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_coverArtFileUUID_fkey" FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_profilePicFileUUID_fkey" FOREIGN KEY ("profilePicFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_coverArtFileUUID_fkey " FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - -- remove the unique constraints from Tracks ALTER TABLE "Tracks" DROP CONSTRAINT "Tracks_trackUUID_key"; ALTER TABLE "Tracks" DROP CONSTRAINT "blockchainId_unique_idx"; diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index eb1829d1387..dc4a10d7195 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -19,10 +19,6 @@ module.exports = { // Add composite unique constraint on (cnodeUserUUID, clock) to Files await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) - // Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables - // (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) - // await addCompositePrimaryKeysToAudiusUsersAndTracks(queryInterface, Sequelize, transaction) - await transaction.commit() }, diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index afe246e0634..21d99fced75 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -124,7 +124,7 @@ module.exports = function (app) { method: 'post', data: { wallet: [walletPublicKey], - creator_node_endpoint: 'http://docker.for.mac.localhost:4000', + creator_node_endpoint: primary, immediate: true, db_only_sync: true } From aa577e8bdfb7ccd1969121added46834f190e90d Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 15:53:59 -0400 Subject: [PATCH 31/53] more TODOs --- creator-node/src/routes/nodeSync.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 16eb683cd9e..97a7355079a 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -314,7 +314,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync // TODO - replace this check with a clock check (!!!) const latestBlockNumber = cnodeUser.latestBlockNumber if ((fetchedLatestBlockNumber === -1 && latestBlockNumber !== -1) || - (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber <= latestBlockNumber) + (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber < /*=*/ latestBlockNumber) //TODO put the = back in ) { throw new Error(`Imported data is outdated, will not sync. Imported latestBlockNumber \ ${fetchedLatestBlockNumber} Self latestBlockNumber ${latestBlockNumber}`) From 7c9142560e4e13dc8c49ee30421c1248dfa9b900 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 16:09:31 -0400 Subject: [PATCH 32/53] remove session tokens as part of sync --- creator-node/src/routes/nodeSync.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 97a7355079a..691260457ba 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -358,6 +358,13 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync }) req.logger.info(redisKey, `numClockRecordsDeleted ${numClockRecordsDeleted}`) + // TODO - should we have this? + const numSessionTokensDeleted = await models.SessionToken.destroy({ + where: { cnodeUserUUID }, + transaction + }) + req.logger.info(redisKey, `numSessionTokensDeleted ${numSessionTokensDeleted}`) + // Delete cnodeUser entry await cnodeUser.destroy({ transaction }) req.logger.info(redisKey, `deleted cnodeUserEntry`) From deae29e3f452c42741450afc661580b85af2b719 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 18:43:53 -0400 Subject: [PATCH 33/53] more fixes --- creator-node/src/routes/nodeSync.js | 44 ++++++++++++-------------- creator-node/src/routes/vectorClock.js | 10 ++++-- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 691260457ba..ab9e3888196 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -186,22 +186,9 @@ module.exports = function (app) { // option to sync just the db records as opposed to db records and files on disk, defaults to false const dbOnlySync = true - if (!immediate) { - req.logger.info('debounce time', config.get('debounceTime')) - // Debounce nodeysnc op - for (let wallet of walletPublicKeys) { - if (wallet in syncQueue) { - clearTimeout(syncQueue[wallet]) - req.logger.info('clear timeout for', wallet, 'time', Date.now()) - } - syncQueue[wallet] = setTimeout( - async () => _nodesync(req, [wallet], creatorNodeEndpoint, dbOnlySync), - config.get('debounceTime') - ) - req.logger.info('set timeout for', wallet, 'time', Date.now()) - } - } else { - await _nodesync(req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync) + let errorObj = await _nodesync(req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync) + if (errorObj) { + return errorResponseServerError(errorObj.message) } return successResponse() })) @@ -226,6 +213,7 @@ module.exports = function (app) { async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync) { const start = Date.now() + let errorObj = null // object to track if the function errored, returned at the end of the function req.logger.info('begin nodesync for ', walletPublicKeys, 'time', start) // ensure access to each wallet, then acquire it for sync. @@ -262,9 +250,11 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync throw new Error(`Malformed response from ${creatorNodeEndpoint}.`) } - // Attempt to connect directly to target CNode's IPFS node. - await _initBootstrapAndRefreshPeers(req, resp.data.ipfsIDObj.addresses, redisKey) - req.logger.info(redisKey, 'IPFS Nodes connected + data export received') + if (!dbOnlySync) { + // Attempt to connect directly to target CNode's IPFS node. + await _initBootstrapAndRefreshPeers(req, resp.data.ipfsIDObj.addresses, redisKey) + req.logger.info(redisKey, 'IPFS Nodes connected + data export received') + } // For each CNodeUser, replace local DB state with retrieved data + fetch + save missing files. for (const fetchedCNodeUser of Object.values(resp.data.cnodeUsers)) { @@ -313,11 +303,14 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync // Ensure imported data has higher blocknumber than already stored. // TODO - replace this check with a clock check (!!!) const latestBlockNumber = cnodeUser.latestBlockNumber - if ((fetchedLatestBlockNumber === -1 && latestBlockNumber !== -1) || - (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber < /*=*/ latestBlockNumber) //TODO put the = back in - ) { - throw new Error(`Imported data is outdated, will not sync. Imported latestBlockNumber \ - ${fetchedLatestBlockNumber} Self latestBlockNumber ${latestBlockNumber}`) + + if (!dbOnlySync) { + if ((fetchedLatestBlockNumber === -1 && latestBlockNumber !== -1) || + (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber < /*=*/ latestBlockNumber) //TODO put the = back in + ) { + throw new Error(`Imported data is outdated, will not sync. Imported latestBlockNumber \ + ${fetchedLatestBlockNumber} Self latestBlockNumber ${latestBlockNumber}`) + } } const cnodeUserUUID = cnodeUser.cnodeUserUUID @@ -476,6 +469,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync } } catch (e) { req.logger.error('Sync Error', e) + errorObj = e } finally { // Release all redis locks for (let wallet of walletPublicKeys) { @@ -485,6 +479,8 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync } req.logger.info(`DURATION SYNC ${Date.now() - start}`) } + + return errorObj } /** Given IPFS node peer addresses, add to bootstrap peers list and manually connect. */ diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 21d99fced75..33990157af0 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -21,10 +21,16 @@ module.exports = function (app) { }) // early exit if cnodeUser not found on primary - if (!cnodeUser) return successResponse('No cnodeUser record found on the primary') + if (!cnodeUser){ + await transaction.commit() + return successResponse('No cnodeUser record found on the primary') + } // early exit if clock values have been added for CNodeUser - if (cnodeUser.clock && cnodeUser.clock > 0) return successResponse({ status: 'Already ran successfully!' }) + if (cnodeUser.clock && cnodeUser.clock > 0){ + await transaction.commit() + return successResponse({ status: 'Already ran successfully!' }) + } // Fetch all data for cnodeUserUUIDs: audiusUsers, tracks, files. let [audiusUsers, tracks, files] = await Promise.all([ From e37f101b2fa3bd0a0428b990a8f028a616ca7bc6 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 19:09:06 -0400 Subject: [PATCH 34/53] env var option to turn off printing sequelize logs --- creator-node/src/config.js | 7 +++++++ creator-node/src/models/index.js | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/creator-node/src/config.js b/creator-node/src/config.js index a6f8f0e9d52..6302cb80739 100644 --- a/creator-node/src/config.js +++ b/creator-node/src/config.js @@ -203,6 +203,13 @@ const config = convict({ env: 'hlsSegmentType', default: 'mpegts' }, + printSequelizeLogs: { + doc: 'If we should print logs from sequelize', + format: Boolean, + env: 'printSequelizeLogs', + default: true + }, + // Transcoding settings transcodingMaxConcurrency: { diff --git a/creator-node/src/models/index.js b/creator-node/src/models/index.js index b7b3c95d569..6cc6fd6e674 100644 --- a/creator-node/src/models/index.js +++ b/creator-node/src/models/index.js @@ -10,7 +10,7 @@ const basename = path.basename(__filename) const db = {} const sequelize = new Sequelize(globalConfig.get('dbUrl'), { - logging: true, + logging: globalConfig.get('printSequelizeLogs'), operatorsAliases: false, pool: { max: 100, From 9c2ceea62657c12428dcbb2cad9c3c414731a1ad Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 21:27:12 -0400 Subject: [PATCH 35/53] more forgiving to sync errors --- creator-node/src/routes/vectorClock.js | 74 +++++++++++++++++++------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 33990157af0..615782027bd 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -26,9 +26,22 @@ module.exports = function (app) { return successResponse('No cnodeUser record found on the primary') } - // early exit if clock values have been added for CNodeUser + // clock values have been added for CNodeUser, check if they're consistent across all nodes before returning success if (cnodeUser.clock && cnodeUser.clock > 0){ await transaction.commit() + // first try/catch is to make sure if the secondaries are not synced. if not, we try to sync + try { + await _checkSecondaryClockValues(secondaries, walletPublicKey, cnodeUser.clock) + } catch (e) { + await _triggerSecondarySyncs(primary, secondaries, walletPublicKey) + } + + // if we kick off a sync and it still not fixed, return with error + try { + await _checkSecondaryClockValues(secondaries, walletPublicKey, cnodeUser.clock) + } catch (e) { + return errorResponseServerError(e) + } return successResponse({ status: 'Already ran successfully!' }) } @@ -120,26 +133,49 @@ module.exports = function (app) { return errorResponseServerError(e.message) } - // trigger secondary syncs here - if (secondaries && secondaries.length > 0) { - await Promise.all(secondaries.map(secondary => { - console.log('calling sync to secondary', secondary) - const axiosReq = { - baseURL: secondary, - url: '/vector_clock_sync', - method: 'post', - data: { - wallet: [walletPublicKey], - creator_node_endpoint: primary, - immediate: true, - db_only_sync: true - } - } - return axios(axiosReq) - })) + try { + // trigger secondary syncs here + await _triggerSecondarySyncs(primary, secondaries, walletPublicKey) + await _checkSecondaryClockValues(secondaries, walletPublicKey, cnodeUser.clock) + } catch (e) { + return errorResponseServerError(e) } - return successResponse() })) } + +async function _checkSecondaryClockValues (secondaries, walletPublicKey, clock) { + const resp = (await Promise.all(secondaries.map(secondary => { + const axiosReq = { + baseURL: secondary, + url: `/sync_status/${walletPublicKey}`, + method: 'get' + } + return axios(axiosReq) + }))).map(r => r.data.data.clockValue) + + resp.map(r => { + if (r !== clock) throw new Error(`Secondaries not in sync with primary [${resp}]`) + }) +} + +async function _triggerSecondarySyncs (primary, secondaries, walletPublicKey) { + if (secondaries && secondaries.length > 0) { + await Promise.all(secondaries.map(secondary => { + console.log('calling sync to secondary', secondary) + const axiosReq = { + baseURL: secondary, + url: '/vector_clock_sync', + method: 'post', + data: { + wallet: [walletPublicKey], + creator_node_endpoint: primary, + immediate: true, + db_only_sync: true + } + } + return axios(axiosReq) + })) + } +} \ No newline at end of file From b6edd0f162a71d80d7d4cde6fdf6a79b443beed4 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 22:07:28 -0400 Subject: [PATCH 36/53] bug fixes --- creator-node/src/routes/vectorClock.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 615782027bd..7ffcbf6f9a7 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -7,6 +7,7 @@ module.exports = function (app) { app.post('/vector_clock_backfill/:wallet', handleResponse(async (req, res, next) => { const walletPublicKey = req.params.wallet const { primary, secondaries } = req.body + let clock = 0 const transaction = await models.sequelize.transaction() try { @@ -66,7 +67,6 @@ module.exports = function (app) { files = [] let clockRecords = [] - let clock = 0 allRecords.map(record => { clock += 1 let clockRecord = { cnodeUserUUID: cnodeUser.cnodeUserUUID, clock, createdAt: record.createdAt } @@ -136,7 +136,7 @@ module.exports = function (app) { try { // trigger secondary syncs here await _triggerSecondarySyncs(primary, secondaries, walletPublicKey) - await _checkSecondaryClockValues(secondaries, walletPublicKey, cnodeUser.clock) + await _checkSecondaryClockValues(secondaries, walletPublicKey, clock) } catch (e) { return errorResponseServerError(e) } @@ -146,6 +146,8 @@ module.exports = function (app) { } async function _checkSecondaryClockValues (secondaries, walletPublicKey, clock) { + if (clock > 25000) clock = 25000 + const resp = (await Promise.all(secondaries.map(secondary => { const axiosReq = { baseURL: secondary, @@ -156,6 +158,7 @@ async function _checkSecondaryClockValues (secondaries, walletPublicKey, clock) }))).map(r => r.data.data.clockValue) resp.map(r => { + if (r !== clock) throw new Error(`Secondaries not in sync with primary [${resp}]`) }) } From e792839af9d0f6c2ac9487ee6f6e778aa0b722da Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 22 Sep 2020 23:09:10 -0400 Subject: [PATCH 37/53] better logging --- creator-node/src/routes/vectorClock.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 7ffcbf6f9a7..bc507210148 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -34,7 +34,7 @@ module.exports = function (app) { try { await _checkSecondaryClockValues(secondaries, walletPublicKey, cnodeUser.clock) } catch (e) { - await _triggerSecondarySyncs(primary, secondaries, walletPublicKey) + await _triggerSecondarySyncs(req, primary, secondaries, walletPublicKey) } // if we kick off a sync and it still not fixed, return with error @@ -82,7 +82,7 @@ module.exports = function (app) { } clockRecords.push(clockRecord) }) - console.log('final clock value', clock) + req.logger.info('final clock value', clock) // delete the existing records await models.AudiusUser.destroy({ @@ -104,7 +104,7 @@ module.exports = function (app) { // chunk files by 10000 records to insert if > 10000 if (files.length > 10000) { for (let i = 0; i <= files.length; i += 10000) { - console.log('writing files from idx', i, i + 10000) + req.logger.info('writing files from idx', i, i + 10000) await models.File.bulkCreate(files.slice(i, i + 10000), { transaction }) } } else { @@ -117,7 +117,7 @@ module.exports = function (app) { if (clockRecords.length > 10000) { for (let i = 0; i <= clockRecords.length; i += 10000) { - console.log('writing clockrecords from idx', i, i + 10000) + req.logger.info('writing clockrecords from idx', i, i + 10000) await models.ClockRecord.bulkCreate(clockRecords.slice(i, i + 10000), { transaction }) } } else { @@ -135,7 +135,7 @@ module.exports = function (app) { try { // trigger secondary syncs here - await _triggerSecondarySyncs(primary, secondaries, walletPublicKey) + await _triggerSecondarySyncs(req, primary, secondaries, walletPublicKey) await _checkSecondaryClockValues(secondaries, walletPublicKey, clock) } catch (e) { return errorResponseServerError(e) @@ -158,15 +158,14 @@ async function _checkSecondaryClockValues (secondaries, walletPublicKey, clock) }))).map(r => r.data.data.clockValue) resp.map(r => { - if (r !== clock) throw new Error(`Secondaries not in sync with primary [${resp}]`) }) } -async function _triggerSecondarySyncs (primary, secondaries, walletPublicKey) { +async function _triggerSecondarySyncs (req, primary, secondaries, walletPublicKey) { if (secondaries && secondaries.length > 0) { await Promise.all(secondaries.map(secondary => { - console.log('calling sync to secondary', secondary) + req.logger.info(`calling sync to secondary for ${secondary} - ${walletPublicKey}`) const axiosReq = { baseURL: secondary, url: '/vector_clock_sync', From 0b15cfa5bd8799edf20b77cb0f61a0352cfd32ea Mon Sep 17 00:00:00 2001 From: SidSethi Date: Wed, 23 Sep 2020 14:21:31 +0000 Subject: [PATCH 38/53] Code cleanup + lint-fix --- creator-node/package-lock.json | 9 +- creator-node/package.json | 7 +- creator-node/scripts/clock-data-migration.js | 202 ------------------ creator-node/scripts/db.js | 51 ----- creator-node/scripts/discprov-users.txt | 10 - .../migrations/20200918150546-add-clock.js | 52 ++--- creator-node/src/config.js | 1 - creator-node/src/dbManager.js | 25 +-- creator-node/src/fileManager.js | 8 +- creator-node/src/middlewares.js | 2 +- creator-node/src/models/audiususer.js | 4 +- creator-node/src/models/clockRecord.js | 6 +- creator-node/src/models/file.js | 10 +- creator-node/src/models/track.js | 6 +- creator-node/src/routes/audiusUsers.js | 4 +- creator-node/src/routes/nodeSync.js | 10 +- creator-node/src/routes/tracks.js | 5 - creator-node/src/routes/users.js | 5 - creator-node/src/routes/vectorClock.js | 15 +- creator-node/test/dbManager.test.js | 23 +- creator-node/test/users.test.js | 3 + 21 files changed, 89 insertions(+), 369 deletions(-) delete mode 100644 creator-node/scripts/clock-data-migration.js delete mode 100644 creator-node/scripts/db.js delete mode 100644 creator-node/scripts/discprov-users.txt diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json index 1d874878aac..8ee47e5a389 100644 --- a/creator-node/package-lock.json +++ b/creator-node/package-lock.json @@ -4072,6 +4072,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", "integrity": "sha1-mo+jb06K1jTjv2tPPIiCVRRS6yA=", + "dev": true, "requires": { "is-object": "~1.0.1", "merge-descriptors": "~1.0.0" @@ -7688,7 +7689,8 @@ "module-not-found-error": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz", - "integrity": "sha1-z4tP9PKWQGdNbN0CsOO8UjwrvcA=" + "integrity": "sha1-z4tP9PKWQGdNbN0CsOO8UjwrvcA=", + "dev": true }, "moment": { "version": "2.24.0", @@ -8835,7 +8837,8 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true }, "path-to-regexp": { "version": "0.1.7", @@ -9305,6 +9308,7 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/proxyquire/-/proxyquire-2.1.3.tgz", "integrity": "sha512-BQWfCqYM+QINd+yawJz23tbBM40VIGXOdDw3X344KcclI/gtBbdWF6SlQ4nK/bYhF9d27KYug9WzljHC6B9Ysg==", + "dev": true, "requires": { "fill-keys": "^1.0.2", "module-not-found-error": "^1.0.1", @@ -9737,6 +9741,7 @@ "version": "1.12.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.12.0.tgz", "integrity": "sha512-B/dOmuoAik5bKcD6s6nXDCjzUKnaDvdkRyAk6rsmsKLipWj4797iothd7jmmUhWTfinVMU+wc56rYKsit2Qy4w==", + "dev": true, "requires": { "path-parse": "^1.0.6" } diff --git a/creator-node/package.json b/creator-node/package.json index 993f06636b0..4390596b289 100644 --- a/creator-node/package.json +++ b/creator-node/package.json @@ -47,7 +47,6 @@ "lodash": "^4.17.15", "multer": "^1.4.0", "pg": "^7.6.1", - "proxyquire": "^2.1.3", "rate-limit-redis": "^1.6.0", "sequelize": "^4.41.2", "shortid": "^2.2.14", @@ -62,7 +61,8 @@ "sinon": "^7.0.0", "standard": "^12.0.1", "fs-extra": "^9.0.1", - "supertest": "^3.3.0" + "supertest": "^3.3.0", + "proxyquire": "^2.1.3" }, "//": { "dependenciesComments": { @@ -80,7 +80,8 @@ "it", "before", "describe", - "afterEach" + "afterEach", + "after" ] } } diff --git a/creator-node/scripts/clock-data-migration.js b/creator-node/scripts/clock-data-migration.js deleted file mode 100644 index a2148906eb4..00000000000 --- a/creator-node/scripts/clock-data-migration.js +++ /dev/null @@ -1,202 +0,0 @@ -// const fs = require('fs-extra') -// const assert = require('assert') -// const axios = require('axios') - -// const initDB = require('./db.js') - -// /** -// * select user_id, wallet, creator_node_endpoint from users -// * where is_current = true and is_creator = true -// * and creator_node_endpoint is not null; -// */ -// const discprovUsersFilePath = './discprov-users-PROD.txt' - -// const usersToRSetMap = {} -// const nodeToUsersMap = {} - -// const endpointToDbUrl = { -// 'http://cn1_creator-node_1:4000': 'postgres://postgres:postgres@127.0.0.1:4432/audius_creator_node', -// 'http://cn2_creator-node_1:4001': 'postgres://postgres:postgres@127.0.0.1:4433/audius_creator_node', -// 'http://cn3_creator-node_1:4002': 'postgres://postgres:postgres@127.0.0.1:4434/audius_creator_node' -// } -// const endpointToDbUrlProd = { -// 'https://creatornode.audius.co': 'postgres://creator_1:postgres@127.0.0.1:5475/audius_creator_node', -// 'https://creatornode2.audius.co': '', -// 'https://creatornode3.audius.co': '' -// } - -// const buildNodeToUsersMap = async () => { -// const fileData = fs.readFileSync(discprovUsersFilePath, 'utf8') -// const fileDataLines = fileData.split('\n') - -// let count = 0 -// for (const fileDataLine of fileDataLines) { -// const [userId, wallet, endpointStr] = fileDataLine.split('\t') -// // console.log(`SIDTEST ROW #${++count}`, userId, wallet, endpointStr) - -// usersToRSetMap[userId] = { wallet, endpointStr } -// const [primary, ...secondaries] = endpointStr.split(',') -// // console.log(` ENDPOINT SPLIT`, primary, secondaries) - -// if (nodeToUsersMap[primary]) { -// nodeToUsersMap[primary].push({ userId, secondaries }) -// } else { -// nodeToUsersMap[primary] = [{ userId, secondaries }] -// } -// } - -// count = 0 -// // for (const user of nodeToUsersMap['https://creatornode.audius.co']) { -// // if (++count > 300) break -// // console.log(`SIDTEST CN1 || USERID: `, user.userId, " || SECONDARIES: ", JSON.stringify(user.secondaries)) -// // } - -// assert.equal(fileDataLines.length, (Object.keys(usersToRSetMap)).length) -// } - -// /** -// for (primary of primaries) -// for (user_id of user_ids) -// update clock state on primary (in transaction) -// select cnodeUserUUID from AudiusUsers for blockchainId -// select all rows in Files, Tracks, AudiusUsers for cnodeUserUUID -// order all data in time order asc -// assign auto-inc clcokval to each row from 1 -// assign final clockval to cnodeUsers row -// force sync secondaries against primary -// */ -// const populateClockVals = async () => { -// const nodes = Object.keys(nodeToUsersMap) - -// // TODO - modify all queries to add SELECT FOR UPDATE and hold lock until commit -// for (const node of nodes) { -// if (node != 'https://creatornode.audius.co') continue -// // init DB instances -// const dbUrl = endpointToDbUrlProd[node] -// const models = await initDB(dbUrl) - -// console.log(`number of users on ${node}: ${nodeToUsersMap[node].length}`) -// for (const { userId, secondaries } of nodeToUsersMap[node]) { -// console.log('\n\n\n\n') -// const start = Date.now() - -// const transaction = await models.sequelize.transaction() - -// try { -// const audiusUsers = await models.AudiusUser.findAll({ -// where: { blockchainId: userId }, -// transaction -// }) - -// let cnodeUserUUID -// if (audiusUsers && audiusUsers.length > 0) { -// cnodeUserUUID = audiusUsers[0].cnodeUserUUID - -// console.log(`userId: ${userId} || audiusUserUUID: ${audiusUsers[0].audiusUserUUID} || cnodeUserUUID: ${cnodeUserUUID}`) - -// // Short circuit if audiusUser already has clock value -// if (audiusUsers[0].clock != null) { -// console.log(`audiusUser already has clock value of ${audiusUsers[0].clock}. Short-circuiting migration`) -// continue -// } - -// const tracks = await models.Track.findAll({ -// where: { cnodeUserUUID }, -// transaction -// }) -// const files = await models.File.findAll({ -// where: { cnodeUserUUID }, -// transaction -// }) - -// // Aggregate all data in array -// const data = [] -// for (const audiusUser of audiusUsers) data.push([audiusUser, `audiusUserUUID ${audiusUser.dataValues.audiusUserUUID}`]) -// for (const track of tracks) data.push([track, `trackUUID ${track.dataValues.trackUUID}`]) -// for (const file of files) data.push([file, `fileUUId ${file.dataValues.fileUUID} - ${file.dataValues.type}`]) - -// // order all rows from Files, Tracks, AudiusUsers by "created" ASC (compares times in milliseconds) -// data.sort((a, b) => { -// const dA = new Date(a[0].dataValues.createdAt).getTime() -// const dB = new Date(b[0].dataValues.createdAt).getTime() -// const diff = dA - dB -// return diff -// }) - -// // Update each data table row with new clock value -// let clockCounter = 0 -// for (const datum of data) { -// await datum[0].update( -// { clock: ++clockCounter }, -// { transaction } -// ) -// } - -// // Update cnodeUser row with final clock value -// const numRowsChanged = await models.CNodeUser.update( -// { clock: clockCounter }, -// { -// where: { cnodeUserUUID }, -// transaction -// } -// ) -// if (!numRowsChanged) { -// throw new Error('CNodeUser update failed') -// } -// } else { -// console.log(`\n\n\nuserId: ${userId} || no audiusUser found`) -// } - -// await transaction.commit() - -// // force sync secondaries -// // if (cnodeUserUUID) { -// // const cnodeUserWallet = usersToRSetMap[userId].wallet -// // await triggerSecondarySyncs(node, secondaries, cnodeUserWallet) -// // await timeout(2000) -// // } -// } catch (e) { -// console.error(`SIDTESTERROR`, e) -// await transaction.rollback() -// } - -// const durationMs = Date.now() - start -// console.log(`USER ROUTE TIME (sec): ${Math.floor(durationMs / 1000)}`) -// } -// } -// console.log('populateClockVals() COMPLETE') -// } - -// buildNodeToUsersMap() -// populateClockVals() - -// /** -// * Tell all secondaries to sync against self. -// * @dev - Is not a middleware so it can be run before responding to client. -// */ -// async function triggerSecondarySyncs (primary, secondaries, wallet) { -// // TODO - throw if resp fails -// // TODO - modify sync to not fail on equal blocknumber -// const resp = await Promise.all(secondaries.map( -// async (secondary) => { -// if (!secondary) return - -// const axiosReq = { -// baseURL: secondary, -// url: '/sync', -// method: 'post', -// data: { -// wallet: [wallet], -// creator_node_endpoint: primary, -// immediate: true -// } -// } -// return axios(axiosReq) -// } -// )) -// } - -// async function timeout (ms) { -// console.log(`starting timeout of ${ms}`) -// return new Promise(resolve => setTimeout(resolve, ms)) -// } diff --git a/creator-node/scripts/db.js b/creator-node/scripts/db.js deleted file mode 100644 index 06bdeef9372..00000000000 --- a/creator-node/scripts/db.js +++ /dev/null @@ -1,51 +0,0 @@ -// 'use strict' - -// const fs = require('fs') -// const path = require('path') -// const Sequelize = require('sequelize') - -// const modelsDirName = path.resolve('../src/models') - -// const basename = path.basename('index.js') - -// const initDB = async (dbUrl) => { -// const db = {} - -// const sequelize = new Sequelize(dbUrl, { -// logging: true, -// operatorsAliases: false, -// // dialectOptions: { -// // idleTimeoutMillis: 500, -// // connectionTimeoutMillis: 500 -// // }, -// pool: { -// max: 100, -// min: 5, -// acquire: 60000, -// idle: 10000 -// } -// }) - -// fs -// .readdirSync(modelsDirName) -// .filter(file => { -// return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js') -// }) -// .forEach(file => { -// const model = sequelize['import'](path.join(modelsDirName, file)) -// db[model.name] = model -// }) - -// Object.keys(db).forEach(modelName => { -// if (db[modelName].associate) { -// db[modelName].associate(db) -// } -// }) - -// db.sequelize = sequelize -// db.Sequelize = Sequelize - -// return db -// } - -// module.exports = initDB diff --git a/creator-node/scripts/discprov-users.txt b/creator-node/scripts/discprov-users.txt deleted file mode 100644 index 77abb86e656..00000000000 --- a/creator-node/scripts/discprov-users.txt +++ /dev/null @@ -1,10 +0,0 @@ -1 0x17ad89ebc6f0e12e963288e60ee81a1e6e181b91 http://cn1_creator-node_1:4000,http://cn3_creator-node_1:4002,http://cn2_creator-node_1:4001 -2 0xbbc0c90c33d865cc577d261cad7a6ab9b3fc8997 http://cn1_creator-node_1:4000,http://cn3_creator-node_1:4002,http://cn2_creator-node_1:4001 -3 0xcb4a77e905c14eb7a327b3fdc22b82f4e71031ef http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 -4 0x8e70d3452d9575f4acdc11fe6d8ebdc100604f0a http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 -5 0x9a5a9e5c80f5045616e6d7962dcac98e94f29147 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 -6 0x9c690258d041d3d35760fe702e9921d73c0cd61f http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 -7 0x8801eba57efad532db64907b182071c79712b5a2 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 -8 0xe5982822f5a4a1c1be67c35762cf56508940b674 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 -9 0x1f491b220e45d2f9cea88a65977c23813d213833 http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 -10 0xf62736cf02e457db2f24f71cb94ad6a86a64036e http://cn3_creator-node_1:4002,http://cn1_creator-node_1:4000,http://cn2_creator-node_1:4001 \ No newline at end of file diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index dc4a10d7195..6a6d6558288 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -10,7 +10,7 @@ module.exports = { const transaction = await queryInterface.sequelize.transaction() // Add 'clock' column to all 4 tables - await addClockColumn(queryInterface, Sequelize, transaction, true) + await addClockColumn(queryInterface, Sequelize, transaction) // Create Clock table await createClockRecordsTable(queryInterface, Sequelize, transaction) @@ -20,55 +20,43 @@ module.exports = { await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) await transaction.commit() - }, - // TODO - down: async (queryInterface, Sequelize) => { - // Remove uniqueness constraints on (cnodeUserUUID, clock) on all 4 tables - await queryInterface.removeConstraint( - 'CNodeUsers', - 'CNodeUsers_unique_constraint_(cnodeUserUUID,clock)' - ) - await queryInterface.removeConstraint( - 'AudiusUsers', - 'AudiusUsers_unique_constraint_(cnodeUserUUID,clock)' - ) - await queryInterface.removeConstraint( - 'Tracks', - 'Tracks_unique_constraint_(cnodeUserUUID,clock)' - ) + /** + * TODO - contents for follow-up migration + * - add non-null constraints to Clock columns + * - addCompositePrimaryKeysToAudiusUsersAndTracks() + */ + }, - // Remove clock columns on all 4 tables - await queryInterface.removeColumn('CNodeUsers', 'clock') - await queryInterface.removeColumn('AudiusUsers', 'clock') - await queryInterface.removeColumn('Tracks', 'clock') - await queryInterface.removeColumn('Files', 'clock') - } + down: async (queryInterface, Sequelize) => { } } -async function addClockColumn (queryInterface, Sequelize, transaction, allowNull) { +// TODO - enforce non-null constraint in follow-up migration +async function addClockColumn (queryInterface, Sequelize, transaction) { await queryInterface.addColumn('CNodeUsers', 'clock', { type: Sequelize.INTEGER, unique: false, - allowNull + allowNull: true }, { transaction }) await queryInterface.addColumn('AudiusUsers', 'clock', { type: Sequelize.INTEGER, unique: false, - allowNull + allowNull: true }, { transaction }) await queryInterface.addColumn('Tracks', 'clock', { type: Sequelize.INTEGER, unique: false, - allowNull + allowNull: true }, { transaction }) await queryInterface.addColumn('Files', 'clock', { type: Sequelize.INTEGER, unique: false, - allowNull + allowNull: true }, { transaction }) } +// TODO - move to and call in follow-up migration +// eslint-disable-next-line no-unused-vars async function addCompositePrimaryKeysToAudiusUsersAndTracks (queryInterface, Sequelize, transaction) { await queryInterface.addConstraint( 'AudiusUsers', @@ -120,11 +108,15 @@ async function addCompositeUniqueConstraints (queryInterface, Sequelize, transac ) } +/** + * TODO - enforce composite foreign key (cnodeUserUUID, clock) on all SourceTables + * - https://stackoverflow.com/questions/9984022/postgres-fk-referencing-composite-pk + */ async function createClockRecordsTable (queryInterface, Sequelize, transaction) { await queryInterface.createTable('ClockRecords', { cnodeUserUUID: { type: Sequelize.UUID, - primaryKey: true, // composite PK with clock + primaryKey: true, // composite primary key (cnodeUserUUID, clock) unique: false, allowNull: false, references: { @@ -136,7 +128,7 @@ async function createClockRecordsTable (queryInterface, Sequelize, transaction) }, clock: { type: Sequelize.INTEGER, - primaryKey: true, // composite PK with cnodeUserUUID + primaryKey: true, // composite primary key (cnodeUserUUID, clock) unique: false, allowNull: false }, diff --git a/creator-node/src/config.js b/creator-node/src/config.js index 6302cb80739..e974776f433 100644 --- a/creator-node/src/config.js +++ b/creator-node/src/config.js @@ -210,7 +210,6 @@ const config = convict({ default: true }, - // Transcoding settings transcodingMaxConcurrency: { doc: 'Maximum ffmpeg processes to spawn concurrently. If unset (-1), set to # of CPU cores available', diff --git a/creator-node/src/dbManager.js b/creator-node/src/dbManager.js index 1d91a0cb514..d96ce8b821c 100644 --- a/creator-node/src/dbManager.js +++ b/creator-node/src/dbManager.js @@ -1,29 +1,15 @@ const models = require('./models') const sequelize = models.sequelize -/** - * TODO - add DataTables all composite FK to Clocks table - * - https://stackoverflow.com/questions/9984022/postgres-fk-referencing-composite-pk - */ - class DBManager { /** * Given file insert query object and cnodeUserUUID, inserts new file record in DB * and handles all required clock management. * Steps: - * 1. increments cnodeUser clock value + * 1. increments cnodeUser clock value by 1 * 2. insert new ClockRecord entry with new clock value * 3. insert new Data Table (File, Track, AudiusUser) entry with queryObj and new clock value - * - * After initial increment, clock values are read as subquery without reading into JS to guarantee atomicity - * - * * TODO - flesh out jsdoc - * @param {*} queryObj - * @param {*} cnodeUserUUID - * @param {*} sequelizeTableInstance - * @param {*} transaction - * - * TODO - returns + * In steps 2 and 3, clock values are read as subquery to guarantee atomicity */ static async createNewDataRecord (queryObj, cnodeUserUUID, sequelizeTableInstance, transaction) { // Increment CNodeUser.clock value by 1 @@ -35,7 +21,7 @@ class DBManager { const selectCNodeUserClockSubqueryLiteral = _getSelectCNodeUserClockSubqueryLiteral(cnodeUserUUID) - // Add row in ClockRecords table using clock value from CNodeUser + // Add row in ClockRecords table using new CNodeUser.clock await models.ClockRecord.create({ cnodeUserUUID, clock: selectCNodeUserClockSubqueryLiteral, @@ -46,7 +32,7 @@ class DBManager { queryObj.cnodeUserUUID = cnodeUserUUID queryObj.clock = selectCNodeUserClockSubqueryLiteral - // Create new Data table entry with queryObj + // Create new Data table entry with queryObj using new CNodeUser.clock const file = await sequelizeTableInstance.create(queryObj, { transaction }) return file.dataValues @@ -54,9 +40,8 @@ class DBManager { } /** + * returns string literal `select "clock" from "CNodeUsers" where "cnodeUserUUID" = '${cnodeUserUUID}'` * @dev source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause - * @param {*} cnodeUserUUID - * return string "select * from clock where cnodeuseruuid" */ function _getSelectCNodeUserClockSubqueryLiteral (cnodeUserUUID) { const subquery = sequelize.dialect.QueryGenerator.selectQuery('CNodeUsers', { diff --git a/creator-node/src/fileManager.js b/creator-node/src/fileManager.js index 84b129d56c5..00b0214222e 100644 --- a/creator-node/src/fileManager.js +++ b/creator-node/src/fileManager.js @@ -22,9 +22,9 @@ const ALLOWED_UPLOAD_FILE_EXTENSIONS = config.get('allowedUploadFileExtensions') const AUDIO_MIME_TYPE_REGEX = /audio\/(.*)/ /** - * Adds file to IPFS then saves file to disk under multihash name + * Adds file to IPFS then saves file to disk under /multihash name * - * @dev - only call this function when file is not already stored to disk, else use saveFileToIPFSFromFS() + */ async function saveFileFromBufferToIPFSAndDisk (req, buffer) { // make sure user has authenticated before saving file @@ -45,7 +45,9 @@ async function saveFileFromBufferToIPFSAndDisk (req, buffer) { } /** - * Given file path on disk, adds file to IPFS + re-saves under /multihash. + * Given file path on disk, adds file to IPFS + re-saves under /multihash name + * + * @dev - only call this function when file is already stored to disk, else use saveFileFromBufferToIPFSAndDisk() */ async function saveFileToIPFSFromFS (req, srcPath) { // make sure user has authenticated before saving file diff --git a/creator-node/src/middlewares.js b/creator-node/src/middlewares.js index 1072d885988..16f9daa1e0c 100644 --- a/creator-node/src/middlewares.js +++ b/creator-node/src/middlewares.js @@ -99,7 +99,7 @@ async function triggerSecondarySyncs (req) { try { if (!req.session.nodeIsPrimary || !req.session.creatorNodeEndpoints || !Array.isArray(req.session.creatorNodeEndpoints)) return const [primary, ...secondaries] = req.session.creatorNodeEndpoints - + req.logger.error(`SIDTEST: primary ${primary} calling sync against: ${secondaries}`) await Promise.all(secondaries.map(async secondary => { if (!secondary || !_isFQDN(secondary)) return const axiosReq = { diff --git a/creator-node/src/models/audiususer.js b/creator-node/src/models/audiususer.js index 839322e350c..26f1343945b 100644 --- a/creator-node/src/models/audiususer.js +++ b/creator-node/src/models/audiususer.js @@ -3,12 +3,12 @@ module.exports = (sequelize, DataTypes) => { const AudiusUser = sequelize.define('AudiusUser', { cnodeUserUUID: { type: DataTypes.UUID, - primaryKey: true, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) allowNull: false }, clock: { type: DataTypes.INTEGER, - primaryKey: true, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) allowNull: false }, blockchainId: { diff --git a/creator-node/src/models/clockRecord.js b/creator-node/src/models/clockRecord.js index 8106578f1fb..7d5ac6af84f 100644 --- a/creator-node/src/models/clockRecord.js +++ b/creator-node/src/models/clockRecord.js @@ -1,6 +1,6 @@ 'use strict' module.exports = (sequelize, DataTypes) => { - // reference models/File.js + // TODO - why is this not externally accessible? const SourceTableTypesObj = { AudiusUser: 'AudiusUser', Track: 'Track', @@ -34,6 +34,10 @@ module.exports = (sequelize, DataTypes) => { } }, {}) + /** + * TODO - enforce composite foreign key (cnodeUserUUID, clock) on all SourceTables + * - https://stackoverflow.com/questions/9984022/postgres-fk-referencing-composite-pk + */ ClockRecord.associate = (models) => { ClockRecord.belongsTo(models.CNodeUser, { foreignKey: 'cnodeUserUUID', diff --git a/creator-node/src/models/file.js b/creator-node/src/models/file.js index fb22035210a..3adf6172827 100644 --- a/creator-node/src/models/file.js +++ b/creator-node/src/models/file.js @@ -12,10 +12,10 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.UUID, allowNull: false }, - // only non-null for track files (as opposed to image/metadata files) + // only non-null for track/copy320 files (as opposed to image/metadata files) trackBlockchainId: { type: DataTypes.INTEGER, - allowNull: true // `true` as we use File entries for more than just uploaded tracks + allowNull: true }, multihash: { type: DataTypes.TEXT, @@ -75,6 +75,10 @@ module.exports = (sequelize, DataTypes) => { ] }) + /** + * @dev - there is intentionally no reference from File.trackBlockchainId to Track.blockchainId. This is to + * remove the two-way association between these models + */ File.associate = function (models) { File.belongsTo(models.CNodeUser, { foreignKey: 'cnodeUserUUID', @@ -83,7 +87,7 @@ module.exports = (sequelize, DataTypes) => { }) } - // TODO why no work? + // TODO - why is this not externally accessible? File.TrackTypes = ['track', 'copy320'] File.NonTrackTypes = ['dir', 'image', 'metadata'] diff --git a/creator-node/src/models/track.js b/creator-node/src/models/track.js index 3f47185c5db..5670794961c 100644 --- a/creator-node/src/models/track.js +++ b/creator-node/src/models/track.js @@ -4,12 +4,12 @@ module.exports = (sequelize, DataTypes) => { const Track = sequelize.define('Track', { cnodeUserUUID: { type: DataTypes.UUID, - primaryKey: true, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) allowNull: false }, clock: { type: DataTypes.INTEGER, - primaryKey: true, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) allowNull: false }, blockchainId: { @@ -43,7 +43,7 @@ module.exports = (sequelize, DataTypes) => { targetKey: 'cnodeUserUUID', onDelete: 'RESTRICT' }) - Track.belongsTo(models.File, { // belongsTo, or hasOne + Track.belongsTo(models.File, { foreignKey: 'metadataFileUUID', targetKey: 'fileUUID', onDelete: 'RESTRICT' diff --git a/creator-node/src/routes/audiusUsers.js b/creator-node/src/routes/audiusUsers.js index 4b714b3450e..4dce624d793 100644 --- a/creator-node/src/routes/audiusUsers.js +++ b/creator-node/src/routes/audiusUsers.js @@ -19,7 +19,6 @@ module.exports = function (app) { const cnodeUserUUID = req.session.cnodeUserUUID // Save file from buffer to IPFS and disk - // TODO simplify (object destructuring?) let multihash, dstPath try { const resp = await saveFileFromBufferToIPFSAndDisk(req, metadataBuffer) @@ -41,7 +40,6 @@ module.exports = function (app) { } const file = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, models.File, transaction) fileUUID = file.fileUUID - await transaction.commit() } catch (e) { await transaction.rollback() @@ -93,6 +91,7 @@ module.exports = function (app) { return errorResponseBadRequest(e.message) } + // Record AudiusUser entry + update CNodeUser entry in DB const transaction = await models.sequelize.transaction() try { const createAudiusUserQueryObj = { @@ -105,7 +104,6 @@ module.exports = function (app) { await DBManager.createNewDataRecord(createAudiusUserQueryObj, cnodeUserUUID, models.AudiusUser, transaction) // Update cnodeUser.latestBlockNumber - // TODO - can this be deprecated with new clock logic? await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction }) await transaction.commit() diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index ab9e3888196..61a05ea3de8 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -101,8 +101,8 @@ module.exports = function (app) { // this just overrides the clock value to the max clock we're sending over to the secondary so it knows // there's more data to pull if (cnodeUser.clock > limit) { - console.log("nodeSync.js#export - cnode user clock value is higher than limit, resetting", clockRecords[clockRecords.length-1].clock) - cnodeUser.clock = clockRecords[clockRecords.length-1].clock + console.log('nodeSync.js#export - cnode user clock value is higher than limit, resetting', clockRecords[clockRecords.length - 1].clock) + cnodeUser.clock = clockRecords[clockRecords.length - 1].clock } }) @@ -188,7 +188,7 @@ module.exports = function (app) { let errorObj = await _nodesync(req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync) if (errorObj) { - return errorResponseServerError(errorObj.message) + return errorResponseServerError(errorObj.message) } return successResponse() })) @@ -303,10 +303,10 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync // Ensure imported data has higher blocknumber than already stored. // TODO - replace this check with a clock check (!!!) const latestBlockNumber = cnodeUser.latestBlockNumber - + if (!dbOnlySync) { if ((fetchedLatestBlockNumber === -1 && latestBlockNumber !== -1) || - (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber < /*=*/ latestBlockNumber) //TODO put the = back in + (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber < /* = */ latestBlockNumber) // TODO put the = back in ) { throw new Error(`Imported data is outdated, will not sync. Imported latestBlockNumber \ ${fetchedLatestBlockNumber} Self latestBlockNumber ${latestBlockNumber}`) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 1d16ee569a3..d90be6ec641 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -228,7 +228,6 @@ module.exports = function (app) { const cnodeUserUUID = req.session.cnodeUserUUID // Save file from buffer to IPFS and disk - // TODO simplify let multihash, dstPath try { const resp = await saveFileFromBufferToIPFSAndDisk(req, metadataBuffer) @@ -310,7 +309,6 @@ module.exports = function (app) { return errorResponseServerError(e.message) } - logger.debug('Beginning POST /tracks DB transactions') const transaction = await models.sequelize.transaction() try { const existingTrackEntry = await models.Track.findOne({ @@ -399,7 +397,6 @@ module.exports = function (app) { transaction } ) - logger.error(`\n\n\nnumAffectedRows: ${numAffectedRows}`) if (parseInt(numAffectedRows, 10) !== trackSegmentCIDs.length) { throw new Error('Failed to associate files for every track segment CID.') } @@ -437,7 +434,6 @@ module.exports = function (app) { // Update cnodeUser's latestBlockNumber if higher than previous latestBlockNumber. // TODO - move to subquery to guarantee atomicity. - // TODO - can deprecate with clockwork? const updatedCNodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID }, transaction }) if (!updatedCNodeUser || !updatedCNodeUser.latestBlockNumber) { throw new Error('Issue in retrieving udpatedCnodeUser') @@ -452,7 +448,6 @@ module.exports = function (app) { await cnodeUser.update({ latestBlockNumber: blockNumber }, { transaction }) } - logger.info(`completed POST tracks route`) await transaction.commit() triggerSecondarySyncs(req) return successResponse() diff --git a/creator-node/src/routes/users.js b/creator-node/src/routes/users.js index 7e6b376b3dd..581631d28b4 100644 --- a/creator-node/src/routes/users.js +++ b/creator-node/src/routes/users.js @@ -160,11 +160,6 @@ module.exports = function (app) { app.get('/users/clock_status/:walletPublicKey', handleResponse(async (req, res) => { let walletPublicKey = req.params.walletPublicKey - // TODO - this doesn't work - // if (!ethereumUtils.isValidAddress(walletPublicKey)) { - // return errorResponseBadRequest('Ethereum address is invalid') - // } - walletPublicKey = walletPublicKey.toLowerCase() const cnodeUser = await models.CNodeUser.findOne({ diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 33990157af0..b048cb893a9 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -21,13 +21,13 @@ module.exports = function (app) { }) // early exit if cnodeUser not found on primary - if (!cnodeUser){ + if (!cnodeUser) { await transaction.commit() return successResponse('No cnodeUser record found on the primary') } // early exit if clock values have been added for CNodeUser - if (cnodeUser.clock && cnodeUser.clock > 0){ + if (cnodeUser.clock && cnodeUser.clock > 0) { await transaction.commit() return successResponse({ status: 'Already ran successfully!' }) } @@ -97,11 +97,11 @@ module.exports = function (app) { } else { await models.File.bulkCreate(files, { transaction }) } - + await models.Track.bulkCreate(tracks, { transaction }) - + await models.AudiusUser.bulkCreate(audiusUsers, { transaction }) - + if (clockRecords.length > 10000) { for (let i = 0; i <= clockRecords.length; i += 10000) { console.log('writing clockrecords from idx', i, i + 10000) @@ -110,7 +110,7 @@ module.exports = function (app) { } else { await models.ClockRecord.bulkCreate(clockRecords, { transaction }) } - + await cnodeUser.update({ clock }, { transaction }) await transaction.commit() @@ -136,10 +136,9 @@ module.exports = function (app) { } } return axios(axiosReq) - })) + })) } return successResponse() - })) } diff --git a/creator-node/test/dbManager.test.js b/creator-node/test/dbManager.test.js index 57ece7fd484..764773d4d1a 100644 --- a/creator-node/test/dbManager.test.js +++ b/creator-node/test/dbManager.test.js @@ -71,14 +71,14 @@ describe('Test createNewDataRecord()', () => { assert.strictEqual(cnodeUser.clock, initialClockVal + 1) // Validate ClockRecords table state - let clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }}) + let clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID } }) assert.strictEqual(clockRecords.length, 1) let clockRecord = clockRecords[0].dataValues assert.strictEqual(clockRecord.clock, initialClockVal + 1) assert.strictEqual(clockRecord.sourceTable, sequelizeTableInstance.name) // Validate Files table state - let files = await models.File.findAll({ where: { cnodeUserUUID }}) + let files = await models.File.findAll({ where: { cnodeUserUUID } }) assert.strictEqual(files.length, 1) let file = files[0].dataValues assert.strictEqual(file.clock, initialClockVal + 1) @@ -209,15 +209,15 @@ describe('Test createNewDataRecord()', () => { */ // Validate CNodeUsers table state - cnodeUser = await getCNodeUser(cnodeUserUUID) + const cnodeUser = await getCNodeUser(cnodeUserUUID) assert.strictEqual(cnodeUser.clock, initialClockVal) // Validate ClockRecords table state - clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) + const clockRecords = await models.ClockRecord.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) assert.strictEqual(clockRecords.length, 0) // Validate Files table state - files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) + const files = await models.File.findAll({ where: { cnodeUserUUID }, order: [['createdAt', 'DESC']] }) assert.strictEqual(files.length, 0) }) @@ -232,6 +232,7 @@ describe('Test createNewDataRecord()', () => { const transaction = await models.sequelize.transaction() const arr = _.range(1, numEntries + 1) // [1, 2, ..., numEntries] const createdFilesResp = [] + // eslint-disable-next-line no-unused-vars for await (const i of arr) { const createdFile = await DBManager.createNewDataRecord(createFileQueryObj, cnodeUserUUID, sequelizeTableInstance, transaction) createdFilesResp.push(createdFile) @@ -328,15 +329,15 @@ describe('Test ClockRecord model', () => { '2020-09-21 23:04:06.339 +00:00', '2020-09-21 23:04:06.339 +00:00' );`, - { - replacements: { invalidSourceTable }, - type: 'RAW', - raw: true - } + { + replacements: { invalidSourceTable }, + type: 'RAW', + raw: true + } ) } catch (e) { assert.strictEqual(e.name, 'SequelizeDatabaseError') assert.strictEqual(e.original.message, `invalid input value for enum "enum_ClockRecords_sourceTable": "${invalidSourceTable}"`) } }) -}) \ No newline at end of file +}) diff --git a/creator-node/test/users.test.js b/creator-node/test/users.test.js index 27edd4ca34a..b72b0dc48b1 100644 --- a/creator-node/test/users.test.js +++ b/creator-node/test/users.test.js @@ -11,6 +11,7 @@ const { getLibsMock } = require('./lib/libsMock') describe('test Users', function () { let app, server, ipfsMock, libsMock + /** Setup app + global test vars */ beforeEach(async () => { ipfsMock = getIPFSMock() libsMock = getLibsMock() @@ -225,4 +226,6 @@ describe('test Users', function () { .send({}) .expect(401) }) + + it('TODO - clock_status test', async function () {}) }) From 31aed698124eecc24866b96ab79c8e1117b5cbe8 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 23 Sep 2020 10:53:50 -0400 Subject: [PATCH 39/53] lint and cleanup --- creator-node/src/routes/nodeSync.js | 4 +--- creator-node/src/routes/vectorClock.js | 14 +++++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 61a05ea3de8..5e4a1c57b24 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -182,7 +182,6 @@ module.exports = function (app) { app.post('/vector_clock_sync', handleResponse(async (req, res) => { const walletPublicKeys = req.body.wallet // array const creatorNodeEndpoint = req.body.creator_node_endpoint // string - const immediate = true // option to sync just the db records as opposed to db records and files on disk, defaults to false const dbOnlySync = true @@ -306,7 +305,7 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync if (!dbOnlySync) { if ((fetchedLatestBlockNumber === -1 && latestBlockNumber !== -1) || - (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber < /* = */ latestBlockNumber) // TODO put the = back in + (fetchedLatestBlockNumber !== -1 && fetchedLatestBlockNumber <= latestBlockNumber) ) { throw new Error(`Imported data is outdated, will not sync. Imported latestBlockNumber \ ${fetchedLatestBlockNumber} Self latestBlockNumber ${latestBlockNumber}`) @@ -351,7 +350,6 @@ async function _nodesync (req, walletPublicKeys, creatorNodeEndpoint, dbOnlySync }) req.logger.info(redisKey, `numClockRecordsDeleted ${numClockRecordsDeleted}`) - // TODO - should we have this? const numSessionTokensDeleted = await models.SessionToken.destroy({ where: { cnodeUserUUID }, transaction diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index 3afbb43601e..d0e6c993a93 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -1,5 +1,4 @@ const models = require('../models') -const { sequelize } = require('../models') const { handleResponse, successResponse, errorResponseServerError } = require('../apiHelpers') const axios = require('axios') @@ -53,8 +52,13 @@ module.exports = function (app) { models.File.findAll({ where: { cnodeUserUUID: cnodeUser.cnodeUserUUID }, transaction, raw: true }) ]) - audiusUsers.map(record => record.type = 'AudiusUser') - tracks.map(record => record.type = 'Track') + audiusUsers.forEach(record => { + record.type = 'AudiusUser' + }) + + tracks.forEach(record => { + record.type = 'Track' + }) // if it doesn't have a type it's a file let allRecords = audiusUsers.concat(tracks, files) @@ -155,7 +159,7 @@ async function _checkSecondaryClockValues (secondaries, walletPublicKey, clock) } return axios(axiosReq) }))).map(r => r.data.data.clockValue) - + resp.map(r => { if (r !== clock) throw new Error(`Secondaries not in sync with primary [${resp}]`) }) @@ -179,4 +183,4 @@ async function _triggerSecondarySyncs (req, primary, secondaries, walletPublicKe return axios(axiosReq) })) } -} \ No newline at end of file +} From 768d7ce70741a75f1b9b17745eabe1e076015490 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Wed, 23 Sep 2020 15:06:12 +0000 Subject: [PATCH 40/53] Fix tests + standard mocha config --- creator-node/package.json | 10 ++-------- creator-node/test/dbManager.test.js | 9 ++++++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/creator-node/package.json b/creator-node/package.json index 4390596b289..98caa49146f 100644 --- a/creator-node/package.json +++ b/creator-node/package.json @@ -74,14 +74,8 @@ } }, "standard": { - "globals": [ - "assert", - "beforeEach", - "it", - "before", - "describe", - "afterEach", - "after" + "env": [ + "mocha" ] } } diff --git a/creator-node/test/dbManager.test.js b/creator-node/test/dbManager.test.js index 764773d4d1a..d755e33000b 100644 --- a/creator-node/test/dbManager.test.js +++ b/creator-node/test/dbManager.test.js @@ -23,8 +23,15 @@ describe('Test createNewDataRecord()', () => { let cnodeUserUUID, createFileQueryObj - /** Create cnodeUser + confirm initial clock state + define global vars */ + /** Reset DB state + Create cnodeUser + confirm initial clock state + define global vars */ beforeEach(async () => { + // Wipe all CNodeUsers + dependent data + await models.CNodeUser.destroy({ + where: {}, + truncate: true, + cascade: true // cascades delete to all rows with foreign key on cnodeUser + }) + const resp = await createStarterCNodeUser() cnodeUserUUID = resp.cnodeUserUUID req.session = { cnodeUserUUID } From e9d01c9bc108a93ef3772ca878bd43b30861880a Mon Sep 17 00:00:00 2001 From: SidSethi Date: Wed, 23 Sep 2020 15:07:38 +0000 Subject: [PATCH 41/53] Enable test lint --- creator-node/scripts/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creator-node/scripts/run-tests.sh b/creator-node/scripts/run-tests.sh index 2f69bd66495..4d68056d765 100755 --- a/creator-node/scripts/run-tests.sh +++ b/creator-node/scripts/run-tests.sh @@ -100,7 +100,7 @@ export delegateOwnerWallet="0x1eC723075E67a1a2B6969dC5CfF0C6793cb36D25" export delegatePrivateKey="0xdb527e4d4a2412a443c17e1666764d3bba43e89e61129a35f9abc337ec170a5d" # tests -# run_unit_tests +run_unit_tests run_integration_tests rm -r $storagePath From d59ab3bdd749525eb217d251c2c52204837ef304 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 23 Sep 2020 12:22:34 -0400 Subject: [PATCH 42/53] migration for post data migration --- ...2131913-post_vector_clock_db_migrations.js | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js diff --git a/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js b/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js new file mode 100644 index 00000000000..3dc2d749a81 --- /dev/null +++ b/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js @@ -0,0 +1,85 @@ +'use strict' + +/** + * 20200922131913-post_vector_clock_db_migrations + */ + +module.exports = { + up: async (queryInterface, Sequelize) => { + const transaction = await queryInterface.sequelize.transaction() + + // TODO - delete all danging rows without any clock values + await queryInterface.sequelize.query(` + DELETE FROM "AudiusUsers" WHERE "clock" IS NULL; + DELETE FROM "Tracks" WHERE "clock" IS NULL; + DELETE FROM "Files" WHERE "clock" IS NULL; + UPDATE "CNodeUsers" SET "clock" = 0 WHERE "clock" IS NULL; + `) + + // ALTER TABLE "AudiusUsers" ALTER COLUMN "clock" SET NOT NULL; + // ALTER TABLE "Tracks" ALTER COLUMN "clock" SET NOT NULL; + // ALTER TABLE "Files" ALTER COLUMN "clock" SET NOT NULL; + // ALTER TABLE "CNodeUsers" ALTER COLUMN "clock" SET NOT NULL; + await updateClock(queryInterface, Sequelize, transaction, false) + + // add back in foreign key constraints for AudiusUsers and Tracks + await queryInterface.sequelize.query(` + ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_coverArtFileUUID_fkey" FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_profilePicFileUUID_fkey" FOREIGN KEY ("profilePicFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_coverArtFileUUID_fkey " FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; + `) + + // Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables + // (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) + await addCompositePrimaryKeysToAudiusUsersAndTracks(queryInterface, Sequelize, transaction) + + await transaction.commit() + }, + + // TODO + down: async (queryInterface, Sequelize) => { + + } +} + +async function updateClock (queryInterface, Sequelize, transaction, allowNull) { + await queryInterface.changeColumn('CNodeUsers', 'clock', { + type: Sequelize.INTEGER, + allowNull + }, { transaction }) + await queryInterface.changeColumn('AudiusUsers', 'clock', { + type: Sequelize.INTEGER, + allowNull + }, { transaction }) + await queryInterface.changeColumn('Tracks', 'clock', { + type: Sequelize.INTEGER, + allowNull + }, { transaction }) + await queryInterface.changeColumn('Files', 'clock', { + type: Sequelize.INTEGER, + allowNull + }, { transaction }) +} + +async function addCompositePrimaryKeysToAudiusUsersAndTracks (queryInterface, Sequelize, transaction) { + await queryInterface.addConstraint( + 'AudiusUsers', + { + type: 'PRIMARY KEY', + fields: ['cnodeUserUUID', 'clock'], + name: 'AudiusUsers_primary_key_(cnodeUserUUID,clock)', + transaction + } + ) + await queryInterface.addConstraint( + 'Tracks', + { + type: 'PRIMARY KEY', + fields: ['cnodeUserUUID', 'clock'], + name: 'Tracks_primary_key_(cnodeUserUUID,clock)', + transaction + } + ) +} From a3aa52c407244d5c3e827469d67b3962b1d29081 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 23 Sep 2020 20:48:24 -0400 Subject: [PATCH 43/53] more whitelisted route in user metadata node --- creator-node/src/userNodeMiddleware.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creator-node/src/userNodeMiddleware.js b/creator-node/src/userNodeMiddleware.js index 3a2c8b7e9a4..6ab94ba5282 100644 --- a/creator-node/src/userNodeMiddleware.js +++ b/creator-node/src/userNodeMiddleware.js @@ -3,7 +3,7 @@ const config = require('./config') async function userNodeMiddleware (req, res, next) { const isUserMetadataNode = config.get('isUserMetadataNode') - const userNodeRegex = new RegExp(/(users|version|health_check|image_upload|ipfs|export)/gm) + const userNodeRegex = new RegExp(/(users|version|db_check|health_check|image_upload|ipfs|export|vector_clock_backfill)/gm) if (isUserMetadataNode) { const isValidUrl = userNodeRegex.test(req.url) if (!isValidUrl) { From 3a489f15c8658dcb43ea1c0aac3fe6332df073f4 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Thu, 24 Sep 2020 09:03:08 -0400 Subject: [PATCH 44/53] cleanup --- ...2131913-post_vector_clock_db_migrations.js | 5 ---- .../migrations/20200918150546-add-clock.js | 29 ------------------- creator-node/src/routes/vectorClock.js | 2 +- 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js b/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js index 3dc2d749a81..8f896fd5504 100644 --- a/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js +++ b/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js @@ -8,7 +8,6 @@ module.exports = { up: async (queryInterface, Sequelize) => { const transaction = await queryInterface.sequelize.transaction() - // TODO - delete all danging rows without any clock values await queryInterface.sequelize.query(` DELETE FROM "AudiusUsers" WHERE "clock" IS NULL; DELETE FROM "Tracks" WHERE "clock" IS NULL; @@ -16,10 +15,6 @@ module.exports = { UPDATE "CNodeUsers" SET "clock" = 0 WHERE "clock" IS NULL; `) - // ALTER TABLE "AudiusUsers" ALTER COLUMN "clock" SET NOT NULL; - // ALTER TABLE "Tracks" ALTER COLUMN "clock" SET NOT NULL; - // ALTER TABLE "Files" ALTER COLUMN "clock" SET NOT NULL; - // ALTER TABLE "CNodeUsers" ALTER COLUMN "clock" SET NOT NULL; await updateClock(queryInterface, Sequelize, transaction, false) // add back in foreign key constraints for AudiusUsers and Tracks diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index 6a6d6558288..c53a8351eaa 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -20,12 +20,6 @@ module.exports = { await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) await transaction.commit() - - /** - * TODO - contents for follow-up migration - * - add non-null constraints to Clock columns - * - addCompositePrimaryKeysToAudiusUsersAndTracks() - */ }, down: async (queryInterface, Sequelize) => { } @@ -55,29 +49,6 @@ async function addClockColumn (queryInterface, Sequelize, transaction) { }, { transaction }) } -// TODO - move to and call in follow-up migration -// eslint-disable-next-line no-unused-vars -async function addCompositePrimaryKeysToAudiusUsersAndTracks (queryInterface, Sequelize, transaction) { - await queryInterface.addConstraint( - 'AudiusUsers', - { - type: 'PRIMARY KEY', - fields: ['cnodeUserUUID', 'clock'], - name: 'AudiusUsers_primary_key_(cnodeUserUUID,clock)', - transaction - } - ) - await queryInterface.addConstraint( - 'Tracks', - { - type: 'PRIMARY KEY', - fields: ['cnodeUserUUID', 'clock'], - name: 'Tracks_primary_key_(cnodeUserUUID,clock)', - transaction - } - ) -} - async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) { await queryInterface.addConstraint( 'AudiusUsers', diff --git a/creator-node/src/routes/vectorClock.js b/creator-node/src/routes/vectorClock.js index d0e6c993a93..2ff55948a5c 100644 --- a/creator-node/src/routes/vectorClock.js +++ b/creator-node/src/routes/vectorClock.js @@ -23,7 +23,7 @@ module.exports = function (app) { // early exit if cnodeUser not found on primary if (!cnodeUser) { await transaction.commit() - return successResponse('No cnodeUser record found on the primary') + return successResponse({ status: 'No cnodeUser record found on the primary' }) } // clock values have been added for CNodeUser, check if they're consistent across all nodes before returning success From 5c65c7e4586fb9e47322560df86a670d9c461036 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 24 Sep 2020 14:40:02 +0000 Subject: [PATCH 45/53] Fix /tracks/unlisted + /export cleanup --- creator-node/src/routes/files.js | 20 +++++++++----- creator-node/src/routes/nodeSync.js | 40 +++++++++++++++++---------- creator-node/src/routes/tracks.js | 29 +++++++++++-------- creator-node/test/audiusUsers.test.js | 18 ++++++++---- creator-node/test/nodesync.test.js | 0 creator-node/test/tracks.test.js | 7 +++-- 6 files changed, 75 insertions(+), 39 deletions(-) create mode 100644 creator-node/test/nodesync.test.js diff --git a/creator-node/src/routes/files.js b/creator-node/src/routes/files.js index 3cf45f55580..e694c3b3eda 100644 --- a/creator-node/src/routes/files.js +++ b/creator-node/src/routes/files.js @@ -97,9 +97,12 @@ const getCID = async (req, res) => { } // Don't serve if not found in DB. - const queryResults = await models.File.findOne({ where: { - multihash: CID - } }) + const queryResults = await models.File.findOne({ + where: { + multihash: CID + }, + order: [['clock', 'DESC']] + }) if (!queryResults) { return sendResponse(req, res, errorResponseNotFound(`No valid file found for provided CID: ${CID}`)) } @@ -186,10 +189,13 @@ const getDirCID = async (req, res) => { // Don't serve if not found in DB. // Query for the file based on the dirCID and filename - const queryResults = await models.File.findOne({ where: { - dirMultihash: dirCID, - fileName: filename - } }) + const queryResults = await models.File.findOne({ + where: { + dirMultihash: dirCID, + fileName: filename + }, + order: [['clock', 'DESC']] + }) if (!queryResults) { return sendResponse( req, diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 5e4a1c57b24..dc3cef47f0c 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -25,24 +25,25 @@ module.exports = function (app) { */ app.get('/export', handleResponse(async (req, res) => { // TODO - allow for offsets in the /export - const limit = 25000 const walletPublicKeys = req.query.wallet_public_key // array const dbOnlySync = (req.query.db_only_sync === true || req.query.db_only_sync === 'true') + const MaxClock = 25000 + const transaction = await models.sequelize.transaction() try { // Fetch cnodeUser for each walletPublicKey. const cnodeUsers = await models.CNodeUser.findAll({ where: { walletPublicKey: walletPublicKeys }, transaction, raw: true }) const cnodeUserUUIDs = cnodeUsers.map((cnodeUser) => cnodeUser.cnodeUserUUID) - const cnodeUserUUID = cnodeUserUUIDs[0] // assume we only have a single wallet being exported for now // Fetch all data for cnodeUserUUIDs: audiusUsers, tracks, files, clockRecords. + const [audiusUsers, tracks, files, clockRecords] = await Promise.all([ models.AudiusUser.findAll({ where: { cnodeUserUUID: cnodeUserUUIDs, clock: { - [models.Sequelize.Op.lte]: limit + [models.Sequelize.Op.lte]: MaxClock } }, order: [['clock', 'ASC']], @@ -53,7 +54,7 @@ module.exports = function (app) { where: { cnodeUserUUID: cnodeUserUUIDs, clock: { - [models.Sequelize.Op.lte]: limit + [models.Sequelize.Op.lte]: MaxClock } }, order: [['clock', 'ASC']], @@ -64,7 +65,7 @@ module.exports = function (app) { where: { cnodeUserUUID: cnodeUserUUIDs, clock: { - [models.Sequelize.Op.lte]: limit + [models.Sequelize.Op.lte]: MaxClock } }, order: [['clock', 'ASC']], @@ -75,7 +76,7 @@ module.exports = function (app) { where: { cnodeUserUUID: cnodeUserUUIDs, clock: { - [models.Sequelize.Op.lte]: limit + [models.Sequelize.Op.lte]: MaxClock } }, order: [['clock', 'ASC']], @@ -83,13 +84,14 @@ module.exports = function (app) { raw: true }) ]) + await transaction.commit() /** Bundle all data into cnodeUser objects to maximize import speed. */ const cnodeUsersDict = {} cnodeUsers.forEach(cnodeUser => { - // Add cnodeUserUUID data fields. + // Add cnodeUserUUID data fields cnodeUser['audiusUsers'] = [] cnodeUser['tracks'] = [] cnodeUser['files'] = [] @@ -97,19 +99,28 @@ module.exports = function (app) { cnodeUsersDict[cnodeUser.cnodeUserUUID] = cnodeUser - // TODO - remove this once we no longer have a limit in export + // TODO - remove this once we no longer have a MaxClock in export // this just overrides the clock value to the max clock we're sending over to the secondary so it knows // there's more data to pull - if (cnodeUser.clock > limit) { - console.log('nodeSync.js#export - cnode user clock value is higher than limit, resetting', clockRecords[clockRecords.length - 1].clock) + if (cnodeUser.clock > MaxClock) { + // since clockRecords are returned by clock ASC, clock val at last index is largest clock val + console.log('nodeSync.js#export - cnode user clock value is higher than MaxClock, resetting', clockRecords[clockRecords.length - 1].clock) cnodeUser.clock = clockRecords[clockRecords.length - 1].clock } }) - cnodeUsersDict[cnodeUserUUID]['audiusUsers'] = audiusUsers - cnodeUsersDict[cnodeUserUUID]['tracks'] = tracks - cnodeUsersDict[cnodeUserUUID]['files'] = files - cnodeUsersDict[cnodeUserUUID]['clockRecords'] = clockRecords + audiusUsers.forEach(audiusUser => { + cnodeUsersDict[audiusUser.cnodeUserUUID]['audiusUsers'].push(audiusUser) + }) + tracks.forEach(track => { + cnodeUsersDict[track.cnodeUserUUID]['tracks'].push(track) + }) + files.forEach(file => { + cnodeUsersDict[file.cnodeUserUUID]['files'].push(file) + }) + clockRecords.forEach(clockRecord => { + cnodeUsersDict[clockRecord.cnodeUserUUID]['clockRecords'].push(clockRecord) + }) // Expose ipfs node's peer ID. const ipfs = req.app.get('ipfsAPI') @@ -138,6 +149,7 @@ module.exports = function (app) { })) } } + return successResponse({ cnodeUsers: cnodeUsersDict, ipfsIDObj }) } catch (e) { console.error('Error in /export', e) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index d90be6ec641..93bb438eecd 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -558,21 +558,28 @@ module.exports = function (app) { return getCID(req, res) })) - /** List all unlisted tracks for a user */ + /** + * List all unlisted tracks for a user + */ app.get('/tracks/unlisted', authMiddleware, handleResponse(async (req, res) => { - const tracks = await models.Track.findAll({ - where: { - metadataJSON: { - is_unlisted: true - }, - cnodeUserUUID: req.session.cnodeUserUUID + const tracks = (await models.sequelize.query( + `select "metadataJSON"->'title' as "title", "blockchainId" from ( + select "metadataJSON", "blockchainId", row_number() over ( + partition by "blockchainId" order by "clock" desc + ) from "Tracks" + where "cnodeUserUUID" = :cnodeUserUUID + and ("metadataJSON"->>'is_unlisted')::boolean = false + ) as a + where a.row_number = 1;`, + { + replacements: { cnodeUserUUID: req.session.cnodeUserUUID } } - }) + ))[0] return successResponse({ - tracks: tracks.map(t => ({ - title: t.metadataJSON.title, - id: t.blockchainId + tracks: tracks.map(track => ({ + title: track.title, + id: track.blockchainId })) }) })) diff --git a/creator-node/test/audiusUsers.test.js b/creator-node/test/audiusUsers.test.js index 27876f5f153..37e39514b1e 100644 --- a/creator-node/test/audiusUsers.test.js +++ b/creator-node/test/audiusUsers.test.js @@ -16,7 +16,7 @@ const { getIPFSMock } = require('./lib/ipfsMock') const { getLibsMock } = require('./lib/libsMock') const { sortKeys } = require('../src/apiHelpers') -describe('test AudiusUsers', function () { +describe('test AudiusUsers with mocked IPFS', function () { let app, server, session, ipfsMock, libsMock // Will need a '.' in front of storagePath to look at current dir @@ -46,7 +46,7 @@ describe('test AudiusUsers', function () { await server.close() }) - it('creates Audius user', async function () { + it('successfully creates Audius user (POST /audius_users/metadata)', async function () { const metadata = { test: 'field1' } ipfsMock.add.twice().withArgs(Buffer.from(JSON.stringify(metadata))) ipfsMock.pin.add.once().withArgs('testCIDLink') @@ -62,7 +62,7 @@ describe('test AudiusUsers', function () { } }) - it('completes Audius user creation', async function () { + it('successfully completes Audius user creation (POST /audius_users/metadata -> POST /audius_users)', async function () { const metadata = { test: 'field1' } ipfsMock.add.twice().withArgs(Buffer.from(JSON.stringify(metadata))) @@ -90,7 +90,7 @@ describe('test AudiusUsers', function () { // Below block uses actual ipfsClient (unlike first describe block), hence // another describe block for this purpose // NOTE: these tests mock ipfs client errors; otherwise, for happy path, uses actual ipfsClient -describe('tests /audius_users/metadata metadata upload with actual ipfsClient for happy path', function () { +describe('Test AudiusUsers with real IPFS', function () { let app, server, session, libsMock, ipfs // Will need a '.' in front of storagePath to look at current dir @@ -144,7 +144,7 @@ describe('tests /audius_users/metadata metadata upload with actual ipfsClient fo assert.deepStrictEqual(resp.body.error, 'saveFileFromBufferToIPFSAndDisk op failed: Error: ipfs add failed!') }) - it('should successfully add metadata file to filesystem, db, and ipfs', async function () { + it('successfully creates Audius user (POST /audius_users/metadata)', async function () { const metadata = sortKeys({ spaghetti: 'spaghetti' }) const resp = await request(app) .post('/audius_users/metadata') @@ -182,4 +182,12 @@ describe('tests /audius_users/metadata metadata upload with actual ipfsClient fo const metadataBuffer = Buffer.from(JSON.stringify(metadata)) assert.deepStrictEqual(metadataBuffer.compare(ipfsResp), 0) }) + + it('TODO - successfully completes Audius user creation (POST /audius_users/metadata -> POST /audius_users)', async function () { + + }) + + it('TODO - multiple uploads', async function () { + + }) }) diff --git a/creator-node/test/nodesync.test.js b/creator-node/test/nodesync.test.js new file mode 100644 index 00000000000..e69de29bb2d diff --git a/creator-node/test/tracks.test.js b/creator-node/test/tracks.test.js index a4ece23782a..c96a23605c4 100644 --- a/creator-node/test/tracks.test.js +++ b/creator-node/test/tracks.test.js @@ -21,7 +21,7 @@ const testAudioFilePath = path.resolve(__dirname, 'testTrack.mp3') const testAudioFileWrongFormatPath = path.resolve(__dirname, 'testTrackWrongFormat.jpg') const testAudiusFileNumSegments = 32 -describe('test Tracks', function () { +describe('test Tracks with mocked IPFS', function () { let app, server, session, ipfsMock, libsMock beforeEach(async () => { @@ -390,7 +390,7 @@ describe('test Tracks', function () { }) }) -describe('test /track_content and /tracks/metadata with actual ipfsClient', function () { +describe('test Tracks with real IPFS', function () { let app, server, session, libsMock, ipfs // Will need a '.' in front of storagePath to look at current dir @@ -581,6 +581,9 @@ describe('test /track_content and /tracks/metadata with actual ipfsClient', func const metadataBuffer = Buffer.from(JSON.stringify(metadata)) assert.deepStrictEqual(metadataBuffer.compare(ipfsResp), 0) }) + + // ~~~~~~~~~~~~~~~~~~~~~~~~~ /tracks TESTS ~~~~~~~~~~~~~~~~~~~~~~~~~ + it('TODO - POST /tracks tests', async function () {}) }) /** From 49ccd27f076c109854ce7602a16c192d4fd85bde Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 24 Sep 2020 14:49:15 +0000 Subject: [PATCH 46/53] logger revert + fix unlisted --- creator-node/src/routes/tracks.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 93bb438eecd..1da825788e5 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -13,7 +13,6 @@ const TranscodingQueue = require('../TranscodingQueue') const { getCID } = require('./files') const { decode } = require('../hashids.js') const RehydrateIpfsQueue = require('../RehydrateIpfsQueue') -const { logger } = require('../logging.js') const DBManager = require('../dbManager') module.exports = function (app) { @@ -52,7 +51,7 @@ module.exports = function (app) { segmentFilePaths = transcode[0].filePaths transcodedFilePath = transcode[1].filePath - logger.info(`Time taken in /track_content to re-encode track file: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + req.logger.info(`Time taken in /track_content to re-encode track file: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) } catch (err) { // Prune upload artifacts removeTrackFolder(req, req.fileDir) @@ -68,12 +67,12 @@ module.exports = function (app) { const { multihash, dstPath } = await saveFileToIPFSFromFS(req, segmentAbsolutePath) return { multihash, srcPath: segmentFilePath, dstPath } })) - logger.info(`Time taken in /track_content for saving transcode + segment files to IPFS: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + req.logger.info(`Time taken in /track_content for saving transcode + segment files to IPFS: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) // Retrieve all segment durations as map(segment srcFilePath => segment duration) codeBlockTimeStart = Date.now() const segmentDurations = await getSegmentsDuration(req.fileName, req.file.destination) - logger.info(`Time taken in /track_content to get segment duration: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + req.logger.info(`Time taken in /track_content to get segment duration: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) // For all segments, build array of (segment multihash, segment duration) let trackSegments = segmentFileIPFSResps.map((segmentFileIPFSResp) => { @@ -148,12 +147,12 @@ module.exports = function (app) { return errorResponseServerError(e) } - logger.info(`Time taken in /track_content for DB updates: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) + req.logger.info(`Time taken in /track_content for DB updates: ${Date.now() - codeBlockTimeStart}ms for file ${req.fileName}`) // Prune upload artifacts after success removeTrackFolder(req, req.fileDir) - logger.info(`Time taken in /track_content for full route: ${Date.now() - routeTimeStart}ms for file ${req.fileName}`) + req.logger.info(`Time taken in /track_content for full route: ${Date.now() - routeTimeStart}ms for file ${req.fileName}`) return successResponse({ 'transcodedTrackCID': transcodeFileIPFSResp.multihash, 'transcodedTrackUUID': transcodeFileUUID, @@ -438,7 +437,7 @@ module.exports = function (app) { if (!updatedCNodeUser || !updatedCNodeUser.latestBlockNumber) { throw new Error('Issue in retrieving udpatedCnodeUser') } - logger.info( + req.logger.info( `cnodeuser ${cnodeUserUUID} first latestBlockNumber ${cnodeUser.latestBlockNumber} || \ current latestBlockNumber ${updatedCNodeUser.latestBlockNumber} || \ given blockNumber ${blockNumber}` @@ -546,7 +545,7 @@ module.exports = function (app) { } if (libs.identityService) { - logger.info(`Logging listen for track ${blockchainId} by ${delegateOwnerWallet}`) + req.logger.info(`Logging listen for track ${blockchainId} by ${delegateOwnerWallet}`) // Fire and forget listen recording // TODO: Consider queueing these requests libs.identityService.logTrackListen(blockchainId, delegateOwnerWallet) @@ -568,7 +567,7 @@ module.exports = function (app) { partition by "blockchainId" order by "clock" desc ) from "Tracks" where "cnodeUserUUID" = :cnodeUserUUID - and ("metadataJSON"->>'is_unlisted')::boolean = false + and ("metadataJSON"->>'is_unlisted')::boolean = true ) as a where a.row_number = 1;`, { From 4c180200023c4987b278219283754f54cf9b5929 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 24 Sep 2020 15:53:51 +0000 Subject: [PATCH 47/53] Add pt2 clock migration + fix tests --- .../migrations/20200918150546-add-clock.js | 5 --- ...2131913-post_vector_clock_db_migrations.js | 34 +++++++++++++------ creator-node/src/routes/tracks.js | 2 +- creator-node/test/dbManager.test.js | 23 ++++++++++--- creator-node/test/nodesync.test.js | 14 ++++++++ creator-node/test/tracks.test.js | 2 ++ 6 files changed, 58 insertions(+), 22 deletions(-) rename creator-node/sequelize/{ => migrations}/20200922131913-post_vector_clock_db_migrations.js (70%) diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index c53a8351eaa..bdba69f858e 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -25,7 +25,6 @@ module.exports = { down: async (queryInterface, Sequelize) => { } } -// TODO - enforce non-null constraint in follow-up migration async function addClockColumn (queryInterface, Sequelize, transaction) { await queryInterface.addColumn('CNodeUsers', 'clock', { type: Sequelize.INTEGER, @@ -79,10 +78,6 @@ async function addCompositeUniqueConstraints (queryInterface, Sequelize, transac ) } -/** - * TODO - enforce composite foreign key (cnodeUserUUID, clock) on all SourceTables - * - https://stackoverflow.com/questions/9984022/postgres-fk-referencing-composite-pk - */ async function createClockRecordsTable (queryInterface, Sequelize, transaction) { await queryInterface.createTable('ClockRecords', { cnodeUserUUID: { diff --git a/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js b/creator-node/sequelize/migrations/20200922131913-post_vector_clock_db_migrations.js similarity index 70% rename from creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js rename to creator-node/sequelize/migrations/20200922131913-post_vector_clock_db_migrations.js index 8f896fd5504..f5e35b9d001 100644 --- a/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js +++ b/creator-node/sequelize/migrations/20200922131913-post_vector_clock_db_migrations.js @@ -2,59 +2,71 @@ /** * 20200922131913-post_vector_clock_db_migrations + * + * SCOPE + * - delete all data where clock is still null + update remaining to 0 + * - enforce clock non-null constraints + * - add back in foreign key constraints from AudiusUsers and Tracks to Files + * - Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables */ module.exports = { up: async (queryInterface, Sequelize) => { const transaction = await queryInterface.sequelize.transaction() + // delete all data from DataTables where clock is still null + update remaining cnodeUsers clocks to 0 await queryInterface.sequelize.query(` DELETE FROM "AudiusUsers" WHERE "clock" IS NULL; DELETE FROM "Tracks" WHERE "clock" IS NULL; DELETE FROM "Files" WHERE "clock" IS NULL; UPDATE "CNodeUsers" SET "clock" = 0 WHERE "clock" IS NULL; - `) + `, { transaction }) - await updateClock(queryInterface, Sequelize, transaction, false) + await enforceClockNonNullConstraints(queryInterface, Sequelize, transaction) - // add back in foreign key constraints for AudiusUsers and Tracks + // add back in foreign key constraints from AudiusUsers and Tracks to Files await queryInterface.sequelize.query(` ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_coverArtFileUUID_fkey" FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_profilePicFileUUID_fkey" FOREIGN KEY ("profilePicFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_coverArtFileUUID_fkey " FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - `) + `, { transaction }) // Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables - // (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) + // - (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) await addCompositePrimaryKeysToAudiusUsersAndTracks(queryInterface, Sequelize, transaction) + /** + * TODO - enforce composite foreign key (cnodeUserUUID, clock) on all SourceTables + * - https://stackoverflow.com/questions/9984022/postgres-fk-referencing-composite-pk + */ + await transaction.commit() }, - // TODO + /** TODO */ down: async (queryInterface, Sequelize) => { } } -async function updateClock (queryInterface, Sequelize, transaction, allowNull) { +async function enforceClockNonNullConstraints (queryInterface, Sequelize, transaction) { await queryInterface.changeColumn('CNodeUsers', 'clock', { type: Sequelize.INTEGER, - allowNull + allowNull: false }, { transaction }) await queryInterface.changeColumn('AudiusUsers', 'clock', { type: Sequelize.INTEGER, - allowNull + allowNull: false }, { transaction }) await queryInterface.changeColumn('Tracks', 'clock', { type: Sequelize.INTEGER, - allowNull + allowNull: false }, { transaction }) await queryInterface.changeColumn('Files', 'clock', { type: Sequelize.INTEGER, - allowNull + allowNull: false }, { transaction }) } diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 1da825788e5..41951e15a0f 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -451,7 +451,7 @@ module.exports = function (app) { triggerSecondarySyncs(req) return successResponse() } catch (e) { - logger.error(e.message) + req.logger.error(e.message) await transaction.rollback() return errorResponseServerError(e.message) } diff --git a/creator-node/test/dbManager.test.js b/creator-node/test/dbManager.test.js index d755e33000b..38fdaed9f7d 100644 --- a/creator-node/test/dbManager.test.js +++ b/creator-node/test/dbManager.test.js @@ -3,9 +3,13 @@ const proxyquire = require('proxyquire') const _ = require('lodash') const models = require('../src/models') -const { createStarterCNodeUser } = require('./lib/dataSeeds') const DBManager = require('../src/dbManager') +const blacklistManager = require('../src/blacklistManager') const utils = require('../src/utils') +const { createStarterCNodeUser } = require('./lib/dataSeeds') +const { getApp } = require('./lib/app') +const { getIPFSMock } = require('./lib/ipfsMock') +const { getLibsMock } = require('./lib/libsMock') describe('Test createNewDataRecord()', () => { const req = { @@ -15,16 +19,23 @@ describe('Test createNewDataRecord()', () => { } const getCNodeUser = async (cnodeUserUUID) => { - return (await models.CNodeUser.findOne({ where: { cnodeUserUUID } })).dataValues + const cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID } }) + return cnodeUser.dataValues } const initialClockVal = 0 const timeoutMs = 1000 - let cnodeUserUUID, createFileQueryObj + let cnodeUserUUID, createFileQueryObj, server + + /** Init server to run DB migrations */ + before(async function () { + const appInfo = await getApp(getIPFSMock(), getLibsMock(), blacklistManager) + server = appInfo.server + }) /** Reset DB state + Create cnodeUser + confirm initial clock state + define global vars */ - beforeEach(async () => { + beforeEach(async function () { // Wipe all CNodeUsers + dependent data await models.CNodeUser.destroy({ where: {}, @@ -49,12 +60,14 @@ describe('Test createNewDataRecord()', () => { }) /** Wipe all CNodeUsers + dependent data */ - after(async () => { + after(async function () { await models.CNodeUser.destroy({ where: {}, truncate: true, cascade: true // cascades delete to all rows with foreign key on cnodeUser }) + + await server.close() }) it('Sequential createNewDataRecord - create 2 records', async () => { diff --git a/creator-node/test/nodesync.test.js b/creator-node/test/nodesync.test.js index e69de29bb2d..a0a8e8c1177 100644 --- a/creator-node/test/nodesync.test.js +++ b/creator-node/test/nodesync.test.js @@ -0,0 +1,14 @@ +describe('test nodesync', function () { + it('test /export', function () { + /** + * TODO - mock DB state -> confirm export returns deterministict output + */ + }) + + it('test /sync', function () { + /** + * mock export request obj + ensure all files are avail on test IPFS node + * confirm sync successfully + */ + }) +}) diff --git a/creator-node/test/tracks.test.js b/creator-node/test/tracks.test.js index c96a23605c4..2fd326b161b 100644 --- a/creator-node/test/tracks.test.js +++ b/creator-node/test/tracks.test.js @@ -584,6 +584,8 @@ describe('test Tracks with real IPFS', function () { // ~~~~~~~~~~~~~~~~~~~~~~~~~ /tracks TESTS ~~~~~~~~~~~~~~~~~~~~~~~~~ it('TODO - POST /tracks tests', async function () {}) + + it('TODO - parallel track upload', async function () {}) }) /** From 9b72b41c31f52e113169b493e48e67a85b9256ca Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 24 Sep 2020 17:20:08 +0000 Subject: [PATCH 48/53] Service commands tweak --- service-commands/scripts/hosts.js | 1 + 1 file changed, 1 insertion(+) diff --git a/service-commands/scripts/hosts.js b/service-commands/scripts/hosts.js index 70b9b0b2a56..2911f22c420 100644 --- a/service-commands/scripts/hosts.js +++ b/service-commands/scripts/hosts.js @@ -88,6 +88,7 @@ if (cmd === 'add') { throw new Error('Misconfigured local env.\nEnsure AUDIUS_REMOTE_DEV_HOST has been exported and /etc/hosts file has necessary permissions.') } const hostMappings = SERVICES.map(s => `${REMOTE_DEV_HOST} ${s}`) + hostMappings.push(`${REMOTE_DEV_HOST} ${audius_client}`) lines = [...lines, START_SENTINEL, ...hostMappings, END_SENTINEL, '\n'] writeArrayIntoFile(lines) } From 3aa988bdc477e44f69cc8c3979fec505fb70b6ba Mon Sep 17 00:00:00 2001 From: SidSethi Date: Thu, 24 Sep 2020 22:55:54 +0000 Subject: [PATCH 49/53] Disable post-backfill-migration --- .../20200922131913-post_vector_clock_db_migrations.js | 1 + 1 file changed, 1 insertion(+) rename creator-node/sequelize/{migrations => }/20200922131913-post_vector_clock_db_migrations.js (98%) diff --git a/creator-node/sequelize/migrations/20200922131913-post_vector_clock_db_migrations.js b/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js similarity index 98% rename from creator-node/sequelize/migrations/20200922131913-post_vector_clock_db_migrations.js rename to creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js index f5e35b9d001..3b422effff5 100644 --- a/creator-node/sequelize/migrations/20200922131913-post_vector_clock_db_migrations.js +++ b/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js @@ -13,6 +13,7 @@ module.exports = { up: async (queryInterface, Sequelize) => { const transaction = await queryInterface.sequelize.transaction() + console.log('STARTING MIGRATION') // delete all data from DataTables where clock is still null + update remaining cnodeUsers clocks to 0 await queryInterface.sequelize.query(` From 3191b25963bac092e9a15f175b1e979b2f94db36 Mon Sep 17 00:00:00 2001 From: SidSethi Date: Fri, 25 Sep 2020 15:49:48 +0000 Subject: [PATCH 50/53] nits --- creator-node/compose/env/commonEnv.sh | 2 ++ identity-service/package-lock.json | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/creator-node/compose/env/commonEnv.sh b/creator-node/compose/env/commonEnv.sh index e69de29bb2d..04f0593a880 100644 --- a/creator-node/compose/env/commonEnv.sh +++ b/creator-node/compose/env/commonEnv.sh @@ -0,0 +1,2 @@ +export delegateOwnerWallet=0xc2c87570c40f08282056f24dbfa2a321e1551889 +export delegatePrivateKey=0x98f2052770cf89bafd9f557f7bbe684107c135a520a799c2e962765c8385dd04 \ No newline at end of file diff --git a/identity-service/package-lock.json b/identity-service/package-lock.json index 6f603639b68..bd2ff5a9f72 100644 --- a/identity-service/package-lock.json +++ b/identity-service/package-lock.json @@ -7690,11 +7690,6 @@ } } }, - "node-fetch": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" - }, "node-forge": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", From 7a281f6b81c2f026013600650159245082346c0f Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Sun, 27 Sep 2020 12:45:09 -0400 Subject: [PATCH 51/53] Remove post data db migration --- creator-node/compose/env/commonEnv.sh | 2 - ...2131913-post_vector_clock_db_migrations.js | 93 ------------------- 2 files changed, 95 deletions(-) delete mode 100644 creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js diff --git a/creator-node/compose/env/commonEnv.sh b/creator-node/compose/env/commonEnv.sh index 04f0593a880..e69de29bb2d 100644 --- a/creator-node/compose/env/commonEnv.sh +++ b/creator-node/compose/env/commonEnv.sh @@ -1,2 +0,0 @@ -export delegateOwnerWallet=0xc2c87570c40f08282056f24dbfa2a321e1551889 -export delegatePrivateKey=0x98f2052770cf89bafd9f557f7bbe684107c135a520a799c2e962765c8385dd04 \ No newline at end of file diff --git a/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js b/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js deleted file mode 100644 index 3b422effff5..00000000000 --- a/creator-node/sequelize/20200922131913-post_vector_clock_db_migrations.js +++ /dev/null @@ -1,93 +0,0 @@ -'use strict' - -/** - * 20200922131913-post_vector_clock_db_migrations - * - * SCOPE - * - delete all data where clock is still null + update remaining to 0 - * - enforce clock non-null constraints - * - add back in foreign key constraints from AudiusUsers and Tracks to Files - * - Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables - */ - -module.exports = { - up: async (queryInterface, Sequelize) => { - const transaction = await queryInterface.sequelize.transaction() - console.log('STARTING MIGRATION') - - // delete all data from DataTables where clock is still null + update remaining cnodeUsers clocks to 0 - await queryInterface.sequelize.query(` - DELETE FROM "AudiusUsers" WHERE "clock" IS NULL; - DELETE FROM "Tracks" WHERE "clock" IS NULL; - DELETE FROM "Files" WHERE "clock" IS NULL; - UPDATE "CNodeUsers" SET "clock" = 0 WHERE "clock" IS NULL; - `, { transaction }) - - await enforceClockNonNullConstraints(queryInterface, Sequelize, transaction) - - // add back in foreign key constraints from AudiusUsers and Tracks to Files - await queryInterface.sequelize.query(` - ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_coverArtFileUUID_fkey" FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "AudiusUsers" ADD CONSTRAINT "AudiusUsers_profilePicFileUUID_fkey" FOREIGN KEY ("profilePicFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_coverArtFileUUID_fkey " FOREIGN KEY ("coverArtFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - ALTER TABLE "Tracks" ADD CONSTRAINT "Tracks_metadataFileUUID_fkey" FOREIGN KEY ("metadataFileUUID") REFERENCES "Files" ("fileUUID") ON DELETE RESTRICT; - `, { transaction }) - - // Add composite primary keys on (cnodeUserUUID,clock) to Tracks and AudiusUsers tables - // - (Files already has PK on fileUUID and cnodeUsers already has PK on cnodeUserUUID) - await addCompositePrimaryKeysToAudiusUsersAndTracks(queryInterface, Sequelize, transaction) - - /** - * TODO - enforce composite foreign key (cnodeUserUUID, clock) on all SourceTables - * - https://stackoverflow.com/questions/9984022/postgres-fk-referencing-composite-pk - */ - - await transaction.commit() - }, - - /** TODO */ - down: async (queryInterface, Sequelize) => { - - } -} - -async function enforceClockNonNullConstraints (queryInterface, Sequelize, transaction) { - await queryInterface.changeColumn('CNodeUsers', 'clock', { - type: Sequelize.INTEGER, - allowNull: false - }, { transaction }) - await queryInterface.changeColumn('AudiusUsers', 'clock', { - type: Sequelize.INTEGER, - allowNull: false - }, { transaction }) - await queryInterface.changeColumn('Tracks', 'clock', { - type: Sequelize.INTEGER, - allowNull: false - }, { transaction }) - await queryInterface.changeColumn('Files', 'clock', { - type: Sequelize.INTEGER, - allowNull: false - }, { transaction }) -} - -async function addCompositePrimaryKeysToAudiusUsersAndTracks (queryInterface, Sequelize, transaction) { - await queryInterface.addConstraint( - 'AudiusUsers', - { - type: 'PRIMARY KEY', - fields: ['cnodeUserUUID', 'clock'], - name: 'AudiusUsers_primary_key_(cnodeUserUUID,clock)', - transaction - } - ) - await queryInterface.addConstraint( - 'Tracks', - { - type: 'PRIMARY KEY', - fields: ['cnodeUserUUID', 'clock'], - name: 'Tracks_primary_key_(cnodeUserUUID,clock)', - transaction - } - ) -} From 6a5171ae99385aa14c94b5a0959347bf80ca1d68 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Sun, 27 Sep 2020 14:38:41 -0400 Subject: [PATCH 52/53] logs at start and end of migrations --- .../20200911004845-allow-track-and-audiusUsers-appends.js | 2 ++ creator-node/sequelize/migrations/20200918150546-add-clock.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js index d7329df0198..b92592e60a8 100644 --- a/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js +++ b/creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js @@ -11,6 +11,7 @@ module.exports = { // Remove UNIQUE constraint for audiusUserUUID in AudiusUsers table (no unique constraint on blockchainId) // Add NOT NULL constraint for blockchainId in AudiusUsers table + console.log('STARTING MIGRATION 20200911004845-allow-track-and-audiusUsers-appends') await queryInterface.sequelize.query(` BEGIN; -- replace Files table in place with extra trackBlockchainId column and drops the trackUUID column @@ -62,6 +63,7 @@ module.exports = { COMMIT; `) + console.log('FINISHED MIGRATION 20200911004845-allow-track-and-audiusUsers-appends') }, down: (queryInterface, Sequelize) => { diff --git a/creator-node/sequelize/migrations/20200918150546-add-clock.js b/creator-node/sequelize/migrations/20200918150546-add-clock.js index bdba69f858e..2c070184f8a 100644 --- a/creator-node/sequelize/migrations/20200918150546-add-clock.js +++ b/creator-node/sequelize/migrations/20200918150546-add-clock.js @@ -7,6 +7,7 @@ module.exports = { up: async (queryInterface, Sequelize) => { + console.log('STARTING MIGRATION 20200918150546-add-clock') const transaction = await queryInterface.sequelize.transaction() // Add 'clock' column to all 4 tables @@ -20,6 +21,7 @@ module.exports = { await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) await transaction.commit() + console.log('FINISHED MIGRATION 20200918150546-add-clock') }, down: async (queryInterface, Sequelize) => { } From 93da492b3517b1e4d48c7c4d2cae1883da79e829 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 29 Sep 2020 11:02:35 -0400 Subject: [PATCH 53/53] test fixes --- creator-node/scripts/run-tests.sh | 2 +- creator-node/test/dbManager.test.js | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/creator-node/scripts/run-tests.sh b/creator-node/scripts/run-tests.sh index 346ccf07fde..fa03c13db71 100755 --- a/creator-node/scripts/run-tests.sh +++ b/creator-node/scripts/run-tests.sh @@ -28,7 +28,7 @@ tear_down () { run_unit_tests () { echo Running unit tests... - ./node_modules/mocha/bin/mocha --recursive 'src/**/*.test.js' --timeout 100000 --exit + ./node_modules/mocha/bin/mocha --recursive 'src/**/*.test.js' --exit } run_integration_tests () { diff --git a/creator-node/test/dbManager.test.js b/creator-node/test/dbManager.test.js index 38fdaed9f7d..3f0a6fab172 100644 --- a/creator-node/test/dbManager.test.js +++ b/creator-node/test/dbManager.test.js @@ -59,6 +59,10 @@ describe('Test createNewDataRecord()', () => { } }) + afterEach(async function () { + models.sequelize.removeHook('beforeCreate', 'clockTimeout') + }) + /** Wipe all CNodeUsers + dependent data */ after(async function () { await models.CNodeUser.destroy({ @@ -145,15 +149,14 @@ describe('Test createNewDataRecord()', () => { const numEntries = 5 // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops - const modelsCopy = models - modelsCopy.sequelize.addHook('beforeCreate', async (instance, options) => { + models.sequelize.addHook('beforeCreate', 'clockTimeout', async (instance, options) => { if (instance.constructor.name === 'ClockRecord') { await utils.timeout(timeoutMs) } }) // Replace required models instance with modified models instance - proxyquire('../src/dbManager', { './models': modelsCopy }) + proxyquire('../src/dbManager', { './models': models }) // Make multiple concurrent calls - create a transaction for each call const arr = _.range(1, numEntries + 1) // [1, 2, ..., numEntries] @@ -199,15 +202,14 @@ describe('Test createNewDataRecord()', () => { const numEntries = 5 // Add global sequelize hook to add timeout before ClockRecord.create calls to force concurrent ops - const modelsCopy = models - modelsCopy.sequelize.addHook('beforeCreate', async (instance, options) => { + models.sequelize.addHook('beforeCreate', 'clockTimeout', async (instance, options) => { if (instance.constructor.name === 'ClockRecord') { await utils.timeout(timeoutMs) } }) // Replace required models instance with modified models instance - proxyquire('../src/dbManager', { './models': modelsCopy }) + proxyquire('../src/dbManager', { './models': models }) // Attempt to make multiple concurrent calls, re-using the same transaction each time const transaction = await models.sequelize.transaction()