diff --git a/.gitignore b/.gitignore index 18ddc4b27..ef0c7dd1f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.orig *.DS_Store *.user +*.qtcreator *.idea *.vscode *.cache @@ -28,3 +29,4 @@ fastlane/report.xml CMakeLists.txt.user CMakeUserPresets.json CLAUDE.md +.claude/* diff --git a/app/activeproject.cpp b/app/activeproject.cpp index 77060f0f6..2c589f79a 100644 --- a/app/activeproject.cpp +++ b/app/activeproject.cpp @@ -169,7 +169,7 @@ bool ActiveProject::forceLoad( const QString &filePath, bool force ) if ( cfgFile.exists() && cfgFile.isFile() ) { // import the new configuration, if it exists. - const QString projectId = MerginProjectMetadata::fromCachedJson( CoreUtils::getProjectMetadataPath( projectDir.path() ) ).projectId; + const QString projectId = MerginProjectMetadata::fromCachedJson( CoreUtils::getProjectMetadataPath( projectDir.path() ) ).id; const bool ok = mAuthManager->importAuthenticationConfigsFromXml( cfgFile.filePath(), projectId, true ); CoreUtils::log( "Authentication database", QStringLiteral( "QGIS auth import of %1 configuration(s) : %2" ).arg( mAuthManager->configIds().count() ).arg( ok ? "successful" : "failed" ) ); } diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 2b5e0c91d..4195eaa40 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2695,7 +2695,7 @@ QString TestMerginApi::projectIdFromProjectFullName( MerginApi *api, const QStri { QByteArray data = r->readAll(); MerginProjectMetadata serverProject = MerginProjectMetadata::fromJson( data ); - ret = serverProject.projectId; + ret = serverProject.id; } else { diff --git a/app/variablesmanager.cpp b/app/variablesmanager.cpp index 3ac616723..525a0a3f2 100644 --- a/app/variablesmanager.cpp +++ b/app/variablesmanager.cpp @@ -194,10 +194,10 @@ void VariablesManager::setProjectVariables() { QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mergin_project_version" ), metadata.version ); QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mergin_project_name" ), metadata.name ); - QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mergin_project_full_name" ), mMerginApi->getFullProjectName( metadata.projectNamespace, metadata.name ) ); + QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mergin_project_full_name" ), mMerginApi->getFullProjectName( metadata.workspaceName, metadata.name ) ); QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mm_project_version" ), metadata.version ); QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mm_project_name" ), metadata.name ); - QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mm_project_full_name" ), mMerginApi->getFullProjectName( metadata.projectNamespace, metadata.name ) ); + QgsExpressionContextUtils::setProjectVariable( mCurrentProject, QStringLiteral( "mm_project_full_name" ), mMerginApi->getFullProjectName( metadata.workspaceName, metadata.name ) ); } else { diff --git a/core/localprojectsmanager.cpp b/core/localprojectsmanager.cpp index f3d134fc8..d7767cbf3 100644 --- a/core/localprojectsmanager.cpp +++ b/core/localprojectsmanager.cpp @@ -36,7 +36,7 @@ void LocalProjectsManager::reloadDataDir() if ( metadata.isValid() ) { info.projectName = metadata.name; - info.projectNamespace = metadata.projectNamespace; + info.projectNamespace = metadata.workspaceName; info.localVersion = metadata.version; } else diff --git a/core/merginapi.cpp b/core/merginapi.cpp index ab91e0611..e5742dae3 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -44,6 +44,7 @@ const QSet MerginApi::sIgnoreExtensions = QSet() << "gpkg-shm" const QSet MerginApi::sIgnoreImageExtensions = QSet() << "jpg" << "jpeg" << "png"; const QSet MerginApi::sIgnoreFiles = QSet() << "mergin.json" << ".DS_Store"; const int MerginApi::UPLOAD_CHUNK_SIZE = 10 * 1024 * 1024; // Should be the same as on the server +const QString MerginApi::sTempChunkId = QStringLiteral( "temp_chunk_id" ); const QString MerginApi::sSyncCanceledMessage = QObject::tr( "Synchronisation canceled" ); #ifdef MOBILE_OS const QString MerginApi::CALLBACK_URL = QStringLiteral( "https://hello.merginmaps.com/mobile/sso-redirect" ); @@ -54,6 +55,17 @@ const QHostAddress MerginApi::OAUTH2_LISTEN_ADDRESS = QHostAddress::Null; constexpr int MerginApi::OAUTH2_LISTEN_PORT = 10042; #endif +static MerginFile findFile( const QString &filePath, const QList &files ) +{ + for ( const MerginFile &merginFile : files ) + { + if ( merginFile.path == filePath ) + return merginFile; + } + CoreUtils::log( QStringLiteral( "MerginFile" ), QStringLiteral( "requested findFile() for non-existant file: %1" ).arg( filePath ) ); + return MerginFile(); +} + MerginApi::MerginApi( LocalProjectsManager &localProjects, QObject *parent ) : QObject( parent ) , mLocalProjects( localProjects ) @@ -172,6 +184,313 @@ MerginSubscriptionInfo *MerginApi::subscriptionInfo() const return mSubscriptionInfo; } +bool MerginApi::pushProject( const QString &projectNamespace, const QString &projectName, bool isInitialPush ) +{ + QString projectFullName = getFullProjectName( projectNamespace, projectName ); + // Get project ID + QString projectId = "271311dd-c09f-4bb1-b88e-77a3d54f9980"; // api testing project + + Q_ASSERT( !mTransactionalStatus.contains( projectFullName ) ); + + CoreUtils::log( "push " + projectFullName, "### Starting ###" ); + CoreUtils::log( "push " + projectFullName, "Project ID: " + projectId ); + + bool useV2Push = true; + bool useV1Push = false; + + if ( useV2Push ) + { + CoreUtils::log( "push " + projectFullName, "Using v2 push API" ); + } + else + { + CoreUtils::log( "push " + projectFullName, "Using v1 push API" ); + } + + TransactionStatus transaction; + transaction.isInitialPush = isInitialPush; + transaction.configAllowed = mSupportsSelectiveSync; + transaction.type = TransactionStatus::Push; + + if ( useV1Push ) + { + // + // In v1 push we always need to be on the latest version before the upload starts. + // So we first request project info and then check if we need to pull first. + // + + // TODO: we can get rid of this project info request when the sync loop is implemented + + QNetworkReply *reply = getProjectInfo( projectFullName ); + + if ( reply ) + { + transaction.replyPushProjectInfo = reply; + connect( reply, &QNetworkReply::finished, this, &MerginApi::pushInfoReplyFinished ); + } + else + { + // TODO: error handling - let user know there is no internet connection + CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED to create project info request!" ) ); + return false; + } + + mTransactionalStatus.insert( projectFullName, transaction ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Requesting project info: " ) + reply->request().url().toString() ); + + return true; + } + else + { + // + // In v2 push we don't need to be on the latest version before the upload starts. + // We rely on the sync manager to always call PULL before PUSH. + // + + LocalProject projectInfo = mLocalProjects.projectFromMerginName( projectFullName ); + transaction.projectDir = projectInfo.projectDir; + Q_ASSERT( !transaction.projectDir.isEmpty() ); + + if ( projectInfo.localVersion == -1 ) + { + // TODO: this is project creation + } + else + { + transaction.version = projectInfo.localVersion; + } + + mTransactionalStatus.insert( projectFullName, transaction ); + + preparePushPayload( projectFullName ); + } + + return true; +} + +void MerginApi::pushInfoReplyFinished() +{ + QNetworkReply *r = qobject_cast( sender() ); + Q_ASSERT( r ); + + QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); + + Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); + TransactionStatus &transaction = mTransactionalStatus[projectFullName]; + Q_ASSERT( r == transaction.replyPushProjectInfo ); + + if ( r->error() == QNetworkReply::NoError ) + { + QString url = r->url().toString(); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Downloaded project info." ) ); + QByteArray data = r->readAll(); + + transaction.replyPushProjectInfo->deleteLater(); + transaction.replyPushProjectInfo = nullptr; + + LocalProject projectInfo = mLocalProjects.projectFromMerginName( projectFullName ); + transaction.projectDir = projectInfo.projectDir; + Q_ASSERT( !transaction.projectDir.isEmpty() ); + + // get the latest server version from our reply (we do not update it in LocalProjectsManager though... I guess we don't need to) + MerginProjectMetadata serverProject = MerginProjectMetadata::fromJson( data ); + + // now let's figure a key question: are we on the most recent version of the project + // if we're about to do upload? because if not, we need to do pull first + if ( projectInfo.isValid() && projectInfo.localVersion != -1 && projectInfo.localVersion < serverProject.version ) + { + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Need pull first: local version %1 | server version %2" ) + .arg( projectInfo.localVersion ).arg( serverProject.version ) ); + transaction.pullBeforePush = true; + prepareProjectPull( projectFullName, data ); + return; + } + + transaction.projectMetadata = data; + transaction.version = MerginProjectMetadata::fromJson( data ).version; + + preparePushPayload( projectFullName ); + } + else + { + QString serverMsg = extractServerErrorMsg( r->readAll() ); + if ( r->error() == QNetworkReply::OperationCanceledError ) + serverMsg = sSyncCanceledMessage; + + QString message = QStringLiteral( "Network API error: %1(): %2" ).arg( QStringLiteral( "projectInfo" ), r->errorString() ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED - %1" ).arg( message ) ); + + int httpCode = r->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + emit networkErrorOccurred( serverMsg, QStringLiteral( "Mergin API error: pushInfo" ), httpCode, projectFullName ); + + transaction.replyPushProjectInfo->deleteLater(); + transaction.replyPushProjectInfo = nullptr; + + finishProjectSync( projectFullName, false ); + } +} + +void MerginApi::preparePushPayload( const QString &projectFullName ) +{ + Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); + TransactionStatus &transaction = mTransactionalStatus[projectFullName]; + + LocalProject projectInfo = mLocalProjects.projectFromMerginName( projectFullName ); + transaction.projectDir = projectInfo.projectDir; + Q_ASSERT( !transaction.projectDir.isEmpty() ); + + QList localFiles = getLocalProjectFiles( transaction.projectDir + "/" ); + MerginProjectMetadata oldServerProject = MerginProjectMetadata::fromCachedJson( transaction.projectDir + "/" + sMetadataFile ); + + // Cache mergin-config, since we are on the most recent version, it is sufficient to just read the local version + if ( transaction.configAllowed ) + { + transaction.config = MerginConfig::fromFile( transaction.projectDir + "/" + MerginApi::sMerginConfigFile ); + } + + // Calculate local changes + transaction.diff = compareProjectFiles( + oldServerProject.files, + oldServerProject.files, + localFiles, + transaction.projectDir, + transaction.configAllowed, + transaction.config + ); + + CoreUtils::log( "push " + projectFullName, transaction.diff.dump() ); + + if ( transaction.configAllowed && transaction.config.isValid && transaction.config.selectiveSyncEnabled ) + { + const QString sDir = transaction.config.selectiveSyncDir.isEmpty() ? QStringLiteral( " for entire project" ) : QStringLiteral( ", on path: %1" ).arg( transaction.config.selectiveSyncDir ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Selective sync is enabled%1" ).arg( sDir ) ); + } + else + { + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Selective sync is not enabled" ) ); + } + + if ( !transaction.diff.remoteAdded.isEmpty() || + !transaction.diff.remoteUpdated.isEmpty() || + !transaction.diff.remoteDeleted.isEmpty() || + !transaction.diff.conflictRemoteAddedLocalAdded.isEmpty() || + !transaction.diff.conflictRemoteDeletedLocalUpdated.isEmpty() || + !transaction.diff.conflictRemoteUpdatedLocalDeleted.isEmpty() || + !transaction.diff.conflictRemoteUpdatedLocalUpdated.isEmpty() ) + { + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Detected remote changes when calculating local changes" ) ); + // What to do here? + } + + const bool useV2push = true; + + for ( const QString &filePath : std::as_const( transaction.diff.localAdded ) ) + { + MerginFile file = findFile( filePath, localFiles ); + file.chunks = generateChunkIdsForSize( file.size ); + + if ( mSupportsSelectiveSync && filePath == sMerginConfigFile ) + { + // problem, we changed selective sync config locally, this is undefined and sync must stop + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Selective sync config was added locally, this is undefined and might lead to data loss, aborting push..." ) ); + finishProjectSync( projectFullName, false ); + } + + transaction.pushChanges.added.append( file ); + } + + for ( const QString &filePath : std::as_const( transaction.diff.localUpdated ) ) + { + MerginFile file = findFile( filePath, localFiles ); + file.chunks = generateChunkIdsForSize( file.size ); + + if ( MerginApi::isFileDiffable( filePath ) ) + { + // try to create a diff + QString diffName; + int geodiffRes = GeodiffUtils::createChangeset( transaction.projectDir, filePath, diffName ); + QString diffPath = transaction.projectDir + "/.mergin/" + diffName; + QString basePath = transaction.projectDir + "/.mergin/" + filePath; + + if ( geodiffRes == GEODIFF_SUCCESS ) + { + QByteArray checksumDiff = CoreUtils::calculateChecksum( diffPath ); + + // TODO: this is ugly. our basefile may not need to have the same checksum as the server's + // basefile (because each of them have applied the diff independently) so we have to fake it + QByteArray checksumBase = oldServerProject.fileInfo( filePath ).checksum.toLatin1(); + + file.diffName = diffName; + file.diffChecksum = QString::fromLatin1( checksumDiff.data(), checksumDiff.size() ); + file.diffSize = QFileInfo( diffPath ).size(); + file.diffBaseChecksum = QString::fromLatin1( checksumBase.data(), checksumBase.size() ); + file.chunks = generateChunkIdsForSize( file.diffSize ); + + CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 successful: total size %2 bytes" ).arg( filePath ).arg( file.diffSize ) ); + } + else + { + // TODO: remove the diff file (if exists) + CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 FAILED with error %2 (will do full upload)" ).arg( filePath ).arg( geodiffRes ) ); + } + } + + if ( mSupportsSelectiveSync && filePath == sMerginConfigFile ) + { + // problem, we changed selective sync config locally, this is undefined and sync must stop + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Selective sync config was updated locally, this is undefined and might lead to data loss, aborting push..." ) ); + finishProjectSync( projectFullName, false ); + } + + transaction.pushChanges.updated.append( file ); + } + + for ( QString filePath : transaction.diff.localDeleted ) + { + MerginFile file = findFile( filePath, oldServerProject.files ); + transaction.pushChanges.removed.append( file ); + } + + if ( transaction.pushChanges.added.isEmpty() && transaction.pushChanges.updated.isEmpty() && transaction.pushChanges.removed.isEmpty() ) + { + finishProjectSync( projectFullName, true ); + + // TODO: emit pushFinished( EverythingUploadedState ~ sicko zrobeno ); + + return; + } + + qint64 totalSize = 0; + for ( MerginFile file : transaction.pushChanges.added ) + { + totalSize += file.size; + } + + for ( MerginFile file : transaction.pushChanges.updated ) + { + if ( !file.diffName.isEmpty() ) + { + totalSize += file.diffSize; + } + else + { + totalSize += file.size; + } + } + + transaction.totalSize = totalSize; + transaction.pushQueue = transaction.pushChanges.added + transaction.pushChanges.updated; + + CoreUtils::log( "push " + projectFullName, QStringLiteral( "%1 items to upload (total size %2 bytes)" ) + .arg( transaction.pushQueue.count() ).arg( totalSize ) ); + + + // TODO: Check here if the total number of files to upload is not larger than accepted by the server (100) + // TODO: Check here if file size of the individual files to upload is not larger than accepted by server (5 GB for gpkg, 10 GB for other file types) + + pushStart( projectFullName ); +} + QString MerginApi::listProjects( const QString &searchExpression, const QString &flag, const int page ) { bool authorize = flag != "public"; @@ -602,7 +921,7 @@ void MerginApi::cacheServerConfig() } -void MerginApi::pushFile( const QString &projectFullName, const QString &transactionUUID, MerginFile file, int chunkNo ) +void MerginApi::pushFile( const QString &projectFullName, MerginFile file, int chunkNo ) { if ( !validateAuth() || mApiVersionStatus != MerginApiStatus::OK ) { @@ -612,13 +931,17 @@ void MerginApi::pushFile( const QString &projectFullName, const QString &transac Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); TransactionStatus &transaction = mTransactionalStatus[projectFullName]; - QString chunkID = file.chunks.at( chunkNo ); + QString projectId = "271311dd-c09f-4bb1-b88e-77a3d54f9980"; // api testing project QString filePath; if ( file.diffName.isEmpty() ) + { filePath = transaction.projectDir + "/" + file.path; - else // use diff file instead of full file + } + else // use diff file instead of full file + { filePath = transaction.projectDir + "/.mergin/" + file.diffName; + } QFile f( filePath ); QByteArray data; @@ -629,20 +952,47 @@ void MerginApi::pushFile( const QString &projectFullName, const QString &transac data = f.read( UPLOAD_CHUNK_SIZE ); } + const QString chunkID = file.chunks.at( chunkNo ); + QNetworkRequest request = getDefaultRequest(); - QUrl url( mApiRoot + QStringLiteral( "/v1/project/push/chunk/%1/%2" ).arg( transactionUUID, chunkID ) ); - request.setUrl( url ); + + bool useV2push = true; + + if ( useV2push ) + { + Q_ASSERT( chunkID == sTempChunkId ); + + QUrl url( mApiRoot + QStringLiteral( "/v2/projects/%1/chunks" ).arg( projectId ) ); + request.setUrl( url ); + } + else // v1 push + { + QUrl url( mApiRoot + QStringLiteral( "/v1/project/push/chunk/%1/%2" ).arg( transaction.transactionUUID, chunkID ) ); + request.setUrl( url ); + } + request.setRawHeader( "Content-Type", "application/octet-stream" ); request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); + request.setAttribute( static_cast( AttrChunkNo ), chunkNo ); + request.setAttribute( static_cast( AttrChunkSize ), QVariant::fromValue( data.size() ) ); Q_ASSERT( !transaction.replyPushFile ); transaction.replyPushFile = mManager->post( request, data ); - connect( transaction.replyPushFile, &QNetworkReply::finished, this, &MerginApi::pushFileReplyFinished ); - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Uploading item: " ) + url.toString() ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Uploading file: %1, chunk: %2/%3" ).arg( file.path ).arg( chunkNo ).arg( file.chunks.count() ) ); + + if ( useV2push ) + { + connect( transaction.replyPushFile, &QNetworkReply::finished, this, &MerginApi::pushV2FileReplyFinished ); + } + else + { + connect( transaction.replyPushFile, &QNetworkReply::finished, this, &MerginApi::pushFileReplyFinished ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Upload URL: " ) + request.url().toString() ); + } } -void MerginApi::pushStart( const QString &projectFullName, const QByteArray &json ) +void MerginApi::pushStart( const QString &projectFullName ) { if ( !validateAuth() || mApiVersionStatus != MerginApiStatus::OK ) { @@ -652,17 +1002,75 @@ void MerginApi::pushStart( const QString &projectFullName, const QByteArray &jso Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); TransactionStatus &transaction = mTransactionalStatus[projectFullName]; + bool useV1push = false; + bool useV2push = true; + QString projectId = "271311dd-c09f-4bb1-b88e-77a3d54f9980"; // api testing project + + // + // prepare the request JSON body: + // + // "changes": { + // "added": [...], + // "updated": [...], + // "removed": [...], + // } + // + + QJsonArray added = prepareUploadChangesJSON( transaction.pushChanges.added ); + QJsonArray modified = prepareUploadChangesJSON( transaction.pushChanges.updated ); + QJsonArray removed = prepareUploadChangesJSON( transaction.pushChanges.removed ); + + QJsonObject changes; + changes.insert( "added", added ); + changes.insert( "updated", modified ); + changes.insert( "removed", removed ); + + if ( useV1push ) + { + changes.insert( "renamed", QJsonArray() ); // todo: drop? + } + + QJsonObject json; + json.insert( QStringLiteral( "changes" ), changes ); + json.insert( QStringLiteral( "version" ), QString( "v%1" ).arg( transaction.version ) ); + + if ( useV2push ) + { + json.insert( QStringLiteral( "check_only" ), true ); + } + + QJsonDocument jsonDoc; + jsonDoc.setObject( json ); + QNetworkRequest request = getDefaultRequest(); - QUrl url( mApiRoot + QStringLiteral( "/v1/project/push/%1" ).arg( projectFullName ) ); - request.setUrl( url ); + + if ( useV2push ) + { + QUrl url( mApiRoot + QStringLiteral( "/v2/projects/%1/versions" ).arg( projectId ) ); + request.setUrl( url ); + } + else // if ( useV1push ) + { + QUrl url( mApiRoot + QStringLiteral( "/v1/project/push/%1" ).arg( projectFullName ) ); + request.setUrl( url ); + } + request.setRawHeader( "Content-Type", "application/json" ); request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); Q_ASSERT( !transaction.replyPushStart ); - transaction.replyPushStart = mManager->post( request, json ); - connect( transaction.replyPushStart, &QNetworkReply::finished, this, &MerginApi::pushStartReplyFinished ); + transaction.replyPushStart = mManager->post( request, jsonDoc.toJson( QJsonDocument::Compact ) ); - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Starting push request: " ) + url.toString() ); + if ( useV2push ) + { + connect( transaction.replyPushStart, &QNetworkReply::finished, this, &MerginApi::pushStartV2ReplyFinished ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Starting push check request: " ) + request.url().toString() ); + } + else + { + connect( transaction.replyPushStart, &QNetworkReply::finished, this, &MerginApi::pushStartReplyFinished ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Starting push request: " ) + request.url().toString() ); + } } void MerginApi::cancelPush( const QString &projectFullName ) @@ -782,68 +1190,102 @@ void MerginApi::pushFinish( const QString &projectFullName, const QString &trans CoreUtils::log( "push " + projectFullName, QStringLiteral( "Requesting transaction finish: " ) + transactionUUID ); } -bool MerginApi::pullProject( const QString &projectNamespace, const QString &projectName, bool withAuth ) +void MerginApi::pushV2Finish( const QString &projectFullName ) { - QString projectFullName = getFullProjectName( projectNamespace, projectName ); - bool pullHasStarted = false; - - CoreUtils::log( "pull " + projectFullName, "### Starting ###" ); - - QNetworkReply *reply = getProjectInfo( projectFullName, withAuth ); - if ( reply ) + if ( !validateAuth() || mApiVersionStatus != MerginApiStatus::OK ) { - CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Requesting project info: " ) + reply->request().url().toString() ); + return; + } - Q_ASSERT( !mTransactionalStatus.contains( projectFullName ) ); - mTransactionalStatus.insert( projectFullName, TransactionStatus() ); - mTransactionalStatus[projectFullName].replyPullProjectInfo = reply; - mTransactionalStatus[projectFullName].configAllowed = mSupportsSelectiveSync; - mTransactionalStatus[projectFullName].type = TransactionStatus::Pull; + Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); + TransactionStatus &transaction = mTransactionalStatus[projectFullName]; - emit syncProjectStatusChanged( projectFullName, 0 ); + QString projectId = "271311dd-c09f-4bb1-b88e-77a3d54f9980"; // api testing project - connect( reply, &QNetworkReply::finished, this, &MerginApi::pullInfoReplyFinished ); - pullHasStarted = true; + // let's make sure all chunks are uploaded and have valid IDs + for ( const MerginFile file : transaction.pushChanges.added ) + { + if ( file.chunks.contains( sTempChunkId ) ) + { + // some chunks were not uploaded! + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Some chunks from added files list did not make it to server before push finish!" ) ); + finishProjectSync( projectFullName, false ); + return; + } } - else + + for ( const MerginFile file : transaction.pushChanges.updated ) { - CoreUtils::log( "pull " + projectFullName, QStringLiteral( "FAILED to create project info request!" ) ); + if ( file.chunks.contains( sTempChunkId ) ) + { + // some chunks were not uploaded! + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Some chunks from updated files list did not make it to server before push finish!" ) ); + finishProjectSync( projectFullName, false ); + return; + } } - return pullHasStarted; + QJsonArray added = prepareUploadChangesJSON( transaction.pushChanges.added ); + QJsonArray modified = prepareUploadChangesJSON( transaction.pushChanges.updated ); + QJsonArray removed = prepareUploadChangesJSON( transaction.pushChanges.removed ); + + QJsonObject changes; + changes.insert( "added", added ); + changes.insert( "updated", modified ); + changes.insert( "removed", removed ); + + QJsonObject json; + json.insert( QStringLiteral( "changes" ), changes ); + json.insert( QStringLiteral( "version" ), QString( "v%1" ).arg( transaction.version ) ); + + QJsonDocument jsonDoc; + jsonDoc.setObject( json ); + + QNetworkRequest request = getDefaultRequest(); + + QUrl url( mApiRoot + QStringLiteral( "/v2/projects/%1/versions" ).arg( projectId ) ); + request.setUrl( url ); + + request.setRawHeader( "Content-Type", "application/json" ); + request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); + + Q_ASSERT( !transaction.replyPushFinish ); + transaction.replyPushFinish = mManager->post( request, jsonDoc.toJson( QJsonDocument::Compact ) ); + + connect( transaction.replyPushFinish, &QNetworkReply::finished, this, &MerginApi::pushFinishReplyFinished ); + + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Requesting to create a new project version: %1" ).arg( transaction.version ) ); } -bool MerginApi::pushProject( const QString &projectNamespace, const QString &projectName, bool isInitialPush ) +bool MerginApi::pullProject( const QString &projectNamespace, const QString &projectName, bool withAuth ) { QString projectFullName = getFullProjectName( projectNamespace, projectName ); - bool pushHasStarted = false; + bool pullHasStarted = false; - CoreUtils::log( "push " + projectFullName, "### Starting ###" ); + CoreUtils::log( "pull " + projectFullName, "### Starting ###" ); - QNetworkReply *reply = getProjectInfo( projectFullName ); + QNetworkReply *reply = getProjectInfo( projectFullName, withAuth ); if ( reply ) { - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Requesting project info: " ) + reply->request().url().toString() ); + CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Requesting project info: " ) + reply->request().url().toString() ); - // create entry about pending upload for the project Q_ASSERT( !mTransactionalStatus.contains( projectFullName ) ); mTransactionalStatus.insert( projectFullName, TransactionStatus() ); - mTransactionalStatus[projectFullName].replyPushProjectInfo = reply; - mTransactionalStatus[projectFullName].isInitialPush = isInitialPush; + mTransactionalStatus[projectFullName].replyPullProjectInfo = reply; mTransactionalStatus[projectFullName].configAllowed = mSupportsSelectiveSync; - mTransactionalStatus[projectFullName].type = TransactionStatus::Push; + mTransactionalStatus[projectFullName].type = TransactionStatus::Pull; emit syncProjectStatusChanged( projectFullName, 0 ); - connect( reply, &QNetworkReply::finished, this, &MerginApi::pushInfoReplyFinished ); - pushHasStarted = true; + connect( reply, &QNetworkReply::finished, this, &MerginApi::pullInfoReplyFinished ); + pullHasStarted = true; } else { - CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED to create project info request!" ) ); + CoreUtils::log( "pull " + projectFullName, QStringLiteral( "FAILED to create project info request!" ) ); } - return pushHasStarted; + return pullHasStarted; } void MerginApi::authorize( const QString &login, const QString &password ) @@ -2320,109 +2762,331 @@ void MerginApi::finalizeProjectPull( const QString &projectFullName ) if ( !QFile::remove( CoreUtils::downloadInProgressFilePath( transaction.projectDir ) ) ) CoreUtils::log( QStringLiteral( "sync %1" ).arg( projectFullName ), QStringLiteral( "Failed to remove download in progress file for project name %1" ).arg( projectName ) ); - mLocalProjects.addMerginProject( projectDir, projectNamespace, projectName ); + mLocalProjects.addMerginProject( projectDir, projectNamespace, projectName ); + } + + finishProjectSync( projectFullName, true ); +} + + +void MerginApi::pushStartReplyFinished() +{ + QNetworkReply *r = qobject_cast( sender() ); + Q_ASSERT( r ); + + QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); + + Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); + TransactionStatus &transaction = mTransactionalStatus[projectFullName]; + Q_ASSERT( r == transaction.replyPushStart ); + + if ( r->error() == QNetworkReply::NoError ) + { + QByteArray data = r->readAll(); + + transaction.replyPushStart->deleteLater(); + transaction.replyPushStart = nullptr; + + QList files = transaction.pushQueue; + if ( !files.isEmpty() ) + { + QString transactionUUID; + QJsonDocument doc = QJsonDocument::fromJson( data ); + if ( doc.isObject() ) + { + QJsonObject docObj = doc.object(); + transactionUUID = docObj.value( QStringLiteral( "transaction" ) ).toString(); + transaction.transactionUUID = transactionUUID; + } + + if ( transaction.transactionUUID.isEmpty() ) + { + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Fail! Could not acquire transaction ID" ) ); + finishProjectSync( projectFullName, false ); + } + + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Push request accepted. Transaction ID: " ) + transactionUUID ); + + MerginFile file = files.first(); + pushFile( projectFullName, file ); + emit pushFilesStarted(); + } + else // pushing only files to be removed + { + // we are done here - no upload of chunks, no request to "finish" + // because server immediatelly creates a new version without starting a transaction to upload chunks + + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Push request accepted and no files to upload" ) ); + + transaction.projectMetadata = data; + transaction.version = MerginProjectMetadata::fromJson( data ).version; + + finishProjectSync( projectFullName, true ); + } + } + else + { + QByteArray data = r->readAll(); + QString serverMsg = extractServerErrorMsg( data ); + if ( r->error() == QNetworkReply::OperationCanceledError ) + serverMsg = sSyncCanceledMessage; + + QString code = extractServerErrorCode( data ); + bool showLimitReachedDialog = EnumHelper::isEqual( code, ErrorCode::StorageLimitHit ); + + CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED - %1. %2" ).arg( r->errorString(), serverMsg ) ); + + transaction.replyPushStart->deleteLater(); + transaction.replyPushStart = nullptr; + + if ( showLimitReachedDialog ) + { + const QList files = transaction.pushQueue; + qreal uploadSize = 0; + for ( const MerginFile &f : files ) + { + uploadSize += f.size; + } + emit storageLimitReached( uploadSize ); + + // remove project if it was first time sync - migration + if ( transaction.isInitialPush ) + { + QString projectNamespace, projectName; + extractProjectName( projectFullName, projectNamespace, projectName ); + + detachProjectFromMergin( projectNamespace, projectName, false ); + deleteProject( projectNamespace, projectName, false ); + } + } + else + { + int httpCode = r->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + emit networkErrorOccurred( serverMsg, QStringLiteral( "Mergin API error: pushStartReply" ), httpCode, projectFullName ); + } + finishProjectSync( projectFullName, false ); + } +} + +void MerginApi::pushStartV2ReplyFinished() +{ + QNetworkReply *r = qobject_cast( sender() ); + Q_ASSERT( r ); + + QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); + + Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); + TransactionStatus &transaction = mTransactionalStatus[projectFullName]; + Q_ASSERT( r == transaction.replyPushStart ); + + // + // There are multiple possible outcomes here: + // - no error (204): we continue with push + // - temporary error (409): we still continue with push + // - permanent error (500, 403, storage limit hit and similar): we stop push and abort + // + + + // even for check_only: + // - ProjectLocked (423) (permanent) + // - VersionAlreadyExists (409) (temp) + // - UploadError (422) (permanent) + // - StorageLimitHit (422) (permanent) + // - 404 (permanent) project of workspace is missing + // - 403 (permanent) no permission + // - dry run OK (204) (success dry run) + // + // if not check_only: + // - DataSyncError (422) (permanent) + // - AnotherUploadRunning (409) (temp) + // - UploadError (409 this time?) - version already exists (temp) + // - Project detail (201) (success) + + if ( r->error() != QNetworkReply::NoError ) + { + QByteArray data = r->readAll(); + QString code = extractServerErrorCode( data ); + + bool proceedWithPush = EnumHelper::isEqual( code, ErrorCode::ProjectVersionExists ) || EnumHelper::isEqual( code, ErrorCode::AnotherUploadRunning ); + + if ( proceedWithPush ) + { + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Push check received temporary error: %1, continuing further with push" ).arg( code ) ); + } + else + { + const QString serverMsg = extractServerErrorMsg( data ); + const int httpErrorCode = r->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + + // TODO: parse "error" key from the response JSON and log. Some error objects, like UploadError, contains it. + // TODO: further error handling - storage limit hit, initial project upload (delete the fresh project) + + CoreUtils::log( "push " + projectFullName, QStringLiteral( "CHECK FAILED: %1, %2, %3, %4" ).arg( code ).arg( r->errorString() ).arg( serverMsg ).arg( httpErrorCode ) ); + finishProjectSync( projectFullName, false ); + + return; + } + } + + transaction.replyPushStart->deleteLater(); + transaction.replyPushStart = nullptr; + + if ( transaction.pushQueue.isEmpty() ) + { + // we do not upload anything, just remove, let's skip to creating the version directly + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Nothing to upload, skipping to create new version" ) ); + + pushV2Finish( projectFullName ); + } + + MerginFile file = transaction.pushQueue.first(); + pushFile( projectFullName, file ); + + emit pushFilesStarted(); +} + +void MerginApi::pushV2FileReplyFinished() +{ + QNetworkReply *r = qobject_cast( sender() ); + Q_ASSERT( r ); + + QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); + const int chunkNo = r->request().attribute( static_cast( AttrChunkNo ) ).toInt(); + const qint64 chunkSize = r->request().attribute( static_cast( AttrChunkSize ) ).value(); + Q_ASSERT( chunkNo >= 0 ); + + Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); + TransactionStatus &transaction = mTransactionalStatus[projectFullName]; + Q_ASSERT( r == transaction.replyPushFile ); + + QString chunkID; + + if ( r->error() != QNetworkReply::NoError ) + { + QString serverMsg = extractServerErrorMsg( r->readAll() ); + if ( r->error() == QNetworkReply::OperationCanceledError ) + { + serverMsg = sSyncCanceledMessage; + } + + CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED - %1. %2" ).arg( r->errorString(), serverMsg ) ); + + int httpCode = r->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + emit networkErrorOccurred( serverMsg, QStringLiteral( "Mergin API error: pushFile" ), httpCode, projectFullName ); + + transaction.replyPushFile->deleteLater(); + transaction.replyPushFile = nullptr; + + finishProjectSync( projectFullName, false ); + return; } - finishProjectSync( projectFullName, true ); -} + transaction.replyPushFile->deleteLater(); + transaction.replyPushFile = nullptr; + transaction.transferedSize += chunkSize; + emit syncProjectStatusChanged( projectFullName, transaction.transferedSize / transaction.totalSize ); -void MerginApi::pushStartReplyFinished() -{ - QNetworkReply *r = qobject_cast( sender() ); - Q_ASSERT( r ); + MerginFile processedFile = transaction.pushQueue.first(); - QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); + QJsonDocument resp = QJsonDocument::fromJson( r->readAll() ); + if ( !resp.isObject() ) + { + // Invalid response from the server, abort push, this should not happen though... + CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAIL - Received invalid response from chunk upload, aborting..." ) ); + finishProjectSync( projectFullName, false ); + } - Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); - TransactionStatus &transaction = mTransactionalStatus[projectFullName]; - Q_ASSERT( r == transaction.replyPushStart ); + // let's store the uploaded chunk details + QJsonObject obj = resp.object(); - if ( r->error() == QNetworkReply::NoError ) - { - QByteArray data = r->readAll(); + FileChunkV2 chunk; // todo: do we even need the structure? + chunk.id = obj.value( "id" ).toString(); + chunk.valid_until = obj.value( "valid_until" ).toVariant().toDateTime(); - transaction.replyPushStart->deleteLater(); - transaction.replyPushStart = nullptr; + bool found = false; - QList files = transaction.pushQueue; - if ( !files.isEmpty() ) + // let's try to find the file in the list of added files + for ( MerginFile &merginFile : transaction.pushChanges.added ) + { + if ( merginFile.path == processedFile.path ) { - QString transactionUUID; - QJsonDocument doc = QJsonDocument::fromJson( data ); - if ( doc.isObject() ) + found = true; + + if ( chunkNo >= 0 && chunkNo < merginFile.chunks.count() ) { - QJsonObject docObj = doc.object(); - transactionUUID = docObj.value( QStringLiteral( "transaction" ) ).toString(); - transaction.transactionUUID = transactionUUID; + Q_ASSERT( merginFile.chunks[chunkNo] == sTempChunkId ); + merginFile.chunks[chunkNo] = chunk.id; } - - if ( transaction.transactionUUID.isEmpty() ) + else { - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Fail! Could not acquire transaction ID" ) ); - finishProjectSync( projectFullName, false ); + // something terrible must have happened! + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Could not identify chunk index in added files.." ) ); + finishProjectSync( projectFullName, false ); // todo: is any cleanup needed? } - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Push request accepted. Transaction ID: " ) + transactionUUID ); - - MerginFile file = files.first(); - pushFile( projectFullName, transactionUUID, file ); - emit pushFilesStarted(); + break; } - else // pushing only files to be removed - { - // we are done here - no upload of chunks, no request to "finish" - // because server immediatelly creates a new version without starting a transaction to upload chunks + } - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Push request accepted and no files to upload" ) ); + if ( !found ) + { + // let's try to find the file in the list of updated files, it must be there then + for ( MerginFile &merginFile : transaction.pushChanges.updated ) + { + if ( merginFile.path == processedFile.path ) + { + found = true; - transaction.projectMetadata = data; - transaction.version = MerginProjectMetadata::fromJson( data ).version; + if ( chunkNo >= 0 && chunkNo < merginFile.chunks.count() ) + { + Q_ASSERT( merginFile.chunks[chunkNo] == sTempChunkId ); + merginFile.chunks[chunkNo] = chunk.id; + } + else + { + // something terrible must have happened! + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Could not identify chunk index in updated files.." ) ); + finishProjectSync( projectFullName, false ); + } - finishProjectSync( projectFullName, true ); + break; + } } } - else + + if ( !found ) { - QByteArray data = r->readAll(); - QString serverMsg = extractServerErrorMsg( data ); - if ( r->error() == QNetworkReply::OperationCanceledError ) - serverMsg = sSyncCanceledMessage; + // something terrible must have happened! + CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Could not identify chunk.." ) ); + } + Q_ASSERT( found ); - QString code = extractServerErrorCode( data ); - bool showLimitReachedDialog = EnumHelper::isEqual( code, ErrorCode::StorageLimitHit ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Uploading file: %1, chunk %2/%3 successful, received id: %4, valid until: %5" ).arg( processedFile.path ).arg( chunkNo ).arg( processedFile.chunks.count() ).arg( chunk.id ).arg( chunk.valid_until.toString() ) ); - CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED - %1. %2" ).arg( r->errorString(), serverMsg ) ); + bool fileFullyUploaded = chunkNo == ( processedFile.chunks.size() - 1 ); - transaction.replyPushStart->deleteLater(); - transaction.replyPushStart = nullptr; + if ( fileFullyUploaded ) + { + // TODO: in order to reuse chunks, we need to store them here somewhere - if ( showLimitReachedDialog ) - { - const QList files = transaction.pushQueue; - qreal uploadSize = 0; - for ( const MerginFile &f : files ) - { - uploadSize += f.size; - } - emit storageLimitReached( uploadSize ); + transaction.pushQueue.removeFirst(); - // remove project if it was first time sync - migration - if ( transaction.isInitialPush ) - { - QString projectNamespace, projectName; - extractProjectName( projectFullName, projectNamespace, projectName ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "File %1 is fully uploaded %2" ).arg( processedFile.path ).arg( ( !processedFile.diffName.isEmpty() ? "via diff" : "" ) ) ); - detachProjectFromMergin( projectNamespace, projectName, false ); - deleteProject( projectNamespace, projectName, false ); - } + if ( !transaction.pushQueue.isEmpty() ) + { + MerginFile nextFile = transaction.pushQueue.first(); + pushFile( projectFullName, nextFile ); } else { - int httpCode = r->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); - emit networkErrorOccurred( serverMsg, QStringLiteral( "Mergin API error: pushStartReply" ), httpCode, projectFullName ); + pushV2Finish( projectFullName ); } - finishProjectSync( projectFullName, false ); + } + else + { + pushFile( projectFullName, processedFile, chunkNo + 1 ); } } @@ -2432,6 +3096,9 @@ void MerginApi::pushFileReplyFinished() Q_ASSERT( r ); QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); + const int chunkNo = r->request().attribute( static_cast( AttrChunkNo ) ).toInt(); + const qint64 chunkSize = r->request().attribute( static_cast( AttrChunkSize ) ).value(); + Q_ASSERT( chunkNo >= 0 ); Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); TransactionStatus &transaction = mTransactionalStatus[projectFullName]; @@ -2439,45 +3106,52 @@ void MerginApi::pushFileReplyFinished() QStringList params = ( r->url().toString().split( "/" ) ); QString transactionUUID = params.at( params.length() - 2 ); - QString chunkID = params.at( params.length() - 1 ); Q_ASSERT( transactionUUID == transaction.transactionUUID ); + QString chunkID = params.at( params.length() - 1 ); + if ( r->error() == QNetworkReply::NoError ) { - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Uploaded successfully: " ) + chunkID ); - transaction.replyPushFile->deleteLater(); transaction.replyPushFile = nullptr; - MerginFile currentFile = transaction.pushQueue.first(); - int chunkNo = currentFile.chunks.indexOf( chunkID ); - if ( chunkNo < currentFile.chunks.size() - 1 ) - { - pushFile( projectFullName, transactionUUID, currentFile, chunkNo + 1 ); - } - else - { - transaction.transferedSize += currentFile.size; + transaction.transferedSize += chunkSize; + emit syncProjectStatusChanged( projectFullName, transaction.transferedSize / transaction.totalSize ); + + MerginFile processedFile = transaction.pushQueue.first(); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Uploaded successfully: " ) + chunkID ); - emit syncProjectStatusChanged( projectFullName, transaction.transferedSize / transaction.totalSize ); + Q_ASSERT( chunkNo < processedFile.chunks.count() ); + + bool fileFullyUploaded = chunkNo == ( processedFile.chunks.count() - 1 ); + + if ( fileFullyUploaded ) + { transaction.pushQueue.removeFirst(); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "File %1 is fully uploaded %2" ).arg( processedFile.path ).arg( !processedFile.diffName.isEmpty() ? "via diff" : "" ) ); if ( !transaction.pushQueue.isEmpty() ) { MerginFile nextFile = transaction.pushQueue.first(); - pushFile( projectFullName, transactionUUID, nextFile ); + pushFile( projectFullName, nextFile ); } else { pushFinish( projectFullName, transactionUUID ); } } + else + { + pushFile( projectFullName, processedFile, chunkNo + 1 ); + } } else { QString serverMsg = extractServerErrorMsg( r->readAll() ); if ( r->error() == QNetworkReply::OperationCanceledError ) + { serverMsg = sSyncCanceledMessage; + } CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED - %1. %2" ).arg( r->errorString(), serverMsg ) ); @@ -2864,204 +3538,6 @@ QList MerginApi::itemsForFileDiffs( const MerginFile &file ) return items; } - -static MerginFile findFile( const QString &filePath, const QList &files ) -{ - for ( const MerginFile &merginFile : files ) - { - if ( merginFile.path == filePath ) - return merginFile; - } - CoreUtils::log( QStringLiteral( "MerginFile" ), QStringLiteral( "requested findFile() for non-existant file: %1" ).arg( filePath ) ); - return MerginFile(); -} - - -void MerginApi::pushInfoReplyFinished() -{ - QNetworkReply *r = qobject_cast( sender() ); - Q_ASSERT( r ); - - QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); - - Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); - TransactionStatus &transaction = mTransactionalStatus[projectFullName]; - Q_ASSERT( r == transaction.replyPushProjectInfo ); - - if ( r->error() == QNetworkReply::NoError ) - { - QString url = r->url().toString(); - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Downloaded project info." ) ); - QByteArray data = r->readAll(); - - transaction.replyPushProjectInfo->deleteLater(); - transaction.replyPushProjectInfo = nullptr; - - LocalProject projectInfo = mLocalProjects.projectFromMerginName( projectFullName ); - transaction.projectDir = projectInfo.projectDir; - Q_ASSERT( !transaction.projectDir.isEmpty() ); - - // get the latest server version from our reply (we do not update it in LocalProjectsManager though... I guess we don't need to) - MerginProjectMetadata serverProject = MerginProjectMetadata::fromJson( data ); - - // now let's figure a key question: are we on the most recent version of the project - // if we're about to do upload? because if not, we need to do pull first - if ( projectInfo.isValid() && projectInfo.localVersion != -1 && projectInfo.localVersion < serverProject.version ) - { - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Need pull first: local version %1 | server version %2" ) - .arg( projectInfo.localVersion ).arg( serverProject.version ) ); - transaction.pullBeforePush = true; - prepareProjectPull( projectFullName, data ); - return; - } - - QList localFiles = getLocalProjectFiles( transaction.projectDir + "/" ); - MerginProjectMetadata oldServerProject = MerginProjectMetadata::fromCachedJson( transaction.projectDir + "/" + sMetadataFile ); - - // Cache mergin-config, since we are on the most recent version, it is sufficient to just read the local version - if ( transaction.configAllowed ) - { - transaction.config = MerginConfig::fromFile( transaction.projectDir + "/" + MerginApi::sMerginConfigFile ); - } - - transaction.diff = compareProjectFiles( - oldServerProject.files, - serverProject.files, - localFiles, - transaction.projectDir, - transaction.configAllowed, - transaction.config - ); - - CoreUtils::log( "push " + projectFullName, transaction.diff.dump() ); - - // TODO: make sure there are no remote files to add/update/remove nor conflicts - - QList filesToUpload; - QList addedMerginFiles, updatedMerginFiles, deletedMerginFiles; - QList diffFiles; - for ( QString filePath : transaction.diff.localAdded ) - { - MerginFile merginFile = findFile( filePath, localFiles ); - merginFile.chunks = generateChunkIdsForSize( merginFile.size ); - addedMerginFiles.append( merginFile ); - } - - for ( QString filePath : transaction.diff.localUpdated ) - { - MerginFile merginFile = findFile( filePath, localFiles ); - merginFile.chunks = generateChunkIdsForSize( merginFile.size ); - - if ( MerginApi::isFileDiffable( filePath ) ) - { - // try to create a diff - QString diffName; - int geodiffRes = GeodiffUtils::createChangeset( transaction.projectDir, filePath, diffName ); - QString diffPath = transaction.projectDir + "/.mergin/" + diffName; - QString basePath = transaction.projectDir + "/.mergin/" + filePath; - - if ( geodiffRes == GEODIFF_SUCCESS ) - { - QByteArray checksumDiff = CoreUtils::calculateChecksum( diffPath ); - - // TODO: this is ugly. our basefile may not need to have the same checksum as the server's - // basefile (because each of them have applied the diff independently) so we have to fake it - QByteArray checksumBase = serverProject.fileInfo( filePath ).checksum.toLatin1(); - - merginFile.diffName = diffName; - merginFile.diffChecksum = QString::fromLatin1( checksumDiff.data(), checksumDiff.size() ); - merginFile.diffSize = QFileInfo( diffPath ).size(); - merginFile.chunks = generateChunkIdsForSize( merginFile.diffSize ); - merginFile.diffBaseChecksum = QString::fromLatin1( checksumBase.data(), checksumBase.size() ); - - diffFiles.append( merginFile ); - - CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 successful: total size %2 bytes" ).arg( filePath ).arg( merginFile.diffSize ) ); - } - else - { - // TODO: remove the diff file (if exists) - CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 FAILED with error %2 (will do full upload)" ).arg( filePath ).arg( geodiffRes ) ); - } - } - - updatedMerginFiles.append( merginFile ); - } - - for ( QString filePath : transaction.diff.localDeleted ) - { - MerginFile merginFile = findFile( filePath, serverProject.files ); - deletedMerginFiles.append( merginFile ); - } - - if ( addedMerginFiles.isEmpty() && updatedMerginFiles.isEmpty() && deletedMerginFiles.isEmpty() ) - { - // if nothing has changed, there is no point to even start upload transaction - transaction.projectMetadata = data; - transaction.version = MerginProjectMetadata::fromJson( data ).version; - - finishProjectSync( projectFullName, true ); - return; - } - - QJsonArray added = prepareUploadChangesJSON( addedMerginFiles ); - filesToUpload.append( addedMerginFiles ); - - QJsonArray modified = prepareUploadChangesJSON( updatedMerginFiles ); - filesToUpload.append( updatedMerginFiles ); - - QJsonArray removed = prepareUploadChangesJSON( deletedMerginFiles ); - // removed not in filesToUpload - - QJsonObject changes; - changes.insert( "added", added ); - changes.insert( "removed", removed ); - changes.insert( "updated", modified ); - changes.insert( "renamed", QJsonArray() ); - - qint64 totalSize = 0; - for ( MerginFile file : filesToUpload ) - { - if ( !file.diffName.isEmpty() ) - totalSize += file.diffSize; - else - totalSize += file.size; - } - - CoreUtils::log( "push " + projectFullName, QStringLiteral( "%1 items to upload (total size %2 bytes)" ) - .arg( filesToUpload.count() ).arg( totalSize ) ); - - transaction.totalSize = totalSize; - transaction.pushQueue = filesToUpload; - transaction.pushDiffFiles = diffFiles; - - QJsonObject json; - json.insert( QStringLiteral( "changes" ), changes ); - json.insert( QStringLiteral( "version" ), QString( "v%1" ).arg( serverProject.version ) ); - QJsonDocument jsonDoc; - jsonDoc.setObject( json ); - - pushStart( projectFullName, jsonDoc.toJson( QJsonDocument::Compact ) ); - } - else - { - QString serverMsg = extractServerErrorMsg( r->readAll() ); - if ( r->error() == QNetworkReply::OperationCanceledError ) - serverMsg = sSyncCanceledMessage; - - QString message = QStringLiteral( "Network API error: %1(): %2" ).arg( QStringLiteral( "projectInfo" ), r->errorString() ); - CoreUtils::log( "push " + projectFullName, QStringLiteral( "FAILED - %1" ).arg( message ) ); - - int httpCode = r->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); - emit networkErrorOccurred( serverMsg, QStringLiteral( "Mergin API error: pushInfo" ), httpCode, projectFullName ); - - transaction.replyPushProjectInfo->deleteLater(); - transaction.replyPushProjectInfo = nullptr; - - finishProjectSync( projectFullName, false ); - } -} - void MerginApi::pushFinishReplyFinished() { QNetworkReply *r = qobject_cast( sender() ); @@ -3077,15 +3553,15 @@ void MerginApi::pushFinishReplyFinished() { Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); QByteArray data = r->readAll(); - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Transaction finish accepted" ) ); + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Push finish accepted" ) ); transaction.replyPushFinish->deleteLater(); transaction.replyPushFinish = nullptr; - transaction.projectMetadata = data; - transaction.version = MerginProjectMetadata::fromJson( data ).version; + transaction.projectMetadata = data; // WATCH out, we are now overwriting what we had there before with the new project detail!!! + transaction.version = MerginProjectMetadata::fromJson( data ).version; // WATCH out, we are now overwriting what we had there before with the new project detail!!! - // a new diffable files suppose to have their basefile copies in .mergin + // new diffable files are supposed to have their basefile copies in .mergin for ( QString filePath : transaction.diff.localAdded ) { if ( MerginApi::isFileDiffable( filePath ) ) @@ -3102,9 +3578,13 @@ void MerginApi::pushFinishReplyFinished() } // clean up diff-related files - const auto diffFiles = transaction.pushDiffFiles; - for ( const MerginFile &merginFile : diffFiles ) + for ( const MerginFile &merginFile : transaction.pushChanges.updated ) { + if ( merginFile.diffName.isEmpty() ) + { + continue; + } + QString diffPath = transaction.projectDir + "/.mergin/" + merginFile.diffName; // update basefile (unmodified file that should be equivalent to the server) @@ -3139,9 +3619,12 @@ void MerginApi::pushFinishReplyFinished() emit networkErrorOccurred( serverMsg, QStringLiteral( "Mergin API error: pushFinish" ), httpCode, projectFullName ); // remove temporary diff files - const auto diffFiles = transaction.pushDiffFiles; - for ( const MerginFile &merginFile : diffFiles ) + for ( const MerginFile &merginFile : transaction.pushChanges.updated ) { + if ( merginFile.diffName.isEmpty() ) + { + continue; + } QString diffPath = transaction.projectDir + "/.mergin/" + merginFile.diffName; if ( !QFile::remove( diffPath ) ) CoreUtils::log( "push " + projectFullName, "Failed to remove diff: " + diffPath ); @@ -3637,12 +4120,26 @@ QStringList MerginApi::generateChunkIdsForSize( qint64 fileSize ) // edge case when file is empty, filesize equals zero // manually set one chunk so that file will be synced if ( fileSize <= 0 ) + { noOfChunks = 1; + } + + bool useV2push = true; QStringList chunks; for ( int i = 0; i < noOfChunks; i++ ) { - QString chunkID = CoreUtils::uuidWithoutBraces( QUuid::createUuid() ); + QString chunkID; + + if ( useV2push ) + { + chunkID = sTempChunkId; + } + else + { + chunkID = CoreUtils::uuidWithoutBraces( QUuid::createUuid() ); + } + chunks.append( chunkID ); } return chunks; @@ -3652,13 +4149,13 @@ QJsonArray MerginApi::prepareUploadChangesJSON( const QList &files ) { QJsonArray jsonArray; - for ( MerginFile file : files ) + for ( const MerginFile &file : files ) { QJsonObject fileObject; fileObject.insert( "path", file.path ); fileObject.insert( "size", file.size ); - fileObject.insert( "mtime", file.mtime.toString( Qt::ISODateWithMs ) ); + fileObject.insert( "mtime", file.mtime.toString( Qt::ISODateWithMs ) ); // todo: ignore? if ( !file.diffName.isEmpty() ) { @@ -3677,11 +4174,12 @@ QJsonArray MerginApi::prepareUploadChangesJSON( const QList &files ) } QJsonArray chunksJson; - for ( QString id : file.chunks ) + for ( const QString &id : file.chunks ) { chunksJson.append( id ); } fileObject.insert( "chunks", chunksJson ); + jsonArray.append( fileObject ); } return jsonArray; diff --git a/core/merginapi.h b/core/merginapi.h index 202e042c6..232bfdaa8 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -79,7 +79,7 @@ struct ProjectDiff conflictRemoteUpdatedLocalDeleted == other.conflictRemoteUpdatedLocalDeleted; } - QString dump() const + QString dump() const // TODO: tell if selective sync is used and if it got changed or not { QStringList lines; lines << "--- project diff ---"; @@ -149,6 +149,13 @@ struct TransactionStatus Pull }; + struct PushChanges + { + QList added; + QList updated; + QList removed; + }; + qreal totalSize = 0; //!< total size (in bytes) of files to be pushed or pulled qint64 transferedSize = 0; //!< size (in bytes) of amount of data transferred so far QString transactionUUID; //!< only for push. Initially dummy non-empty string, after server confirms a valid UUID, on finish/cancel it is empty @@ -169,22 +176,22 @@ struct TransactionStatus QList pullTasks; //!< tasks to do at the end of pull when everything has been downloaded bool pullItemsAborting = false; //!< indicates whether we have started to abort requests in replyPullItems - // push-related data + PushChanges pushChanges; + QList pushQueue; //!< pending list of files to push (at the end of transaction it is empty) - QList pushDiffFiles; //!< these are just diff files for push - we don't remove them when pushing chunks (needed for finalization) // retry handling int retryCount = 0; //!< current number of retry attempts for failed network requests static const int MAX_RETRY_COUNT = 5; //!< maximum number of retry attempts for failed network requests QString projectDir; - QByteArray projectMetadata; //!< metadata of the new project (not parsed) + QByteArray projectMetadata; //!< metadata of the new project (not parsed) WATCH OUT - REUSED TWICE NOW -- TODO! bool firstTimeDownload = false; //!< only for update. whether this is first time to download the project (on failure we would also remove the project folder) bool pullBeforePush = false; //!< true when we're first doing update before doing actual upload. Used in sync finalization to figure out whether restart with upload or finish. bool isInitialPush = false; //!< true when we are first time uploading the project - migration to Mergin bool gpkgSchemaChanged = false; //!< true when GPKG schema changes found - int version = -1; //!< version to which we are updating / the version which we have uploaded + int version = -1; //!< version to which we are updating / the version which we have uploaded WATCH OUT - REUSED TWICE NOW -- TODO! ProjectDiff diff; @@ -206,7 +213,9 @@ class ErrorCode { Unknown = 0, ProjectsLimitHit, - StorageLimitHit + StorageLimitHit, + AnotherUploadRunning, + ProjectVersionExists }; Q_ENUM( Value ); }; @@ -308,7 +317,7 @@ class MerginApi: public QObject * \param projectFullName Project's full name to cancel its 4 * \note pushCanceled() signal is emitted when the reply to the cancel request is received */ - Q_INVOKABLE void cancelPush( const QString &projectFullName ); + Q_INVOKABLE void cancelPush( const QString &projectFullName ); //TODO: there is nothing to cancel on v2, is there? .. apart from stop upload /** * Cancels pull either (1) before project data download starts or @@ -390,7 +399,7 @@ class MerginApi: public QObject * \param projectNamespace Project namespace that will be detached from Mergin * \param projectName Project name that will be detached from Mergin */ - Q_INVOKABLE void detachProjectFromMergin( const QString &projectNamespace, const QString &projectName, bool informUser = true ); + Q_INVOKABLE void detachProjectFromMergin( const QString &projectNamespace, const QString &projectName, bool informUser = true ); // TODO: drop /** * Deletes all local projects and then tries to remove user account. @@ -410,6 +419,7 @@ class MerginApi: public QObject static const QString sMetadataFile; static const QString sMetadataFolder; static const QString sMerginConfigFile; + static const QString sTempChunkId; static const QString sDefaultApiRoot; static const QString sSyncCanceledMessage; static const QString sDefaultReportLogUrl; @@ -419,13 +429,13 @@ class MerginApi: public QObject return sDefaultApiRoot; } - static bool isFileDiffable( const QString &fileName ) + static bool isFileDiffable( const QString &fileName ) // todo: go to utils { return fileName.endsWith( ".gpkg" ); } //! Get a list of all files that can be used with geodiff - QStringList projectDiffableFiles( const QString &projectFullName ); + QStringList projectDiffableFiles( const QString &projectFullName ); // todo: drop, definition does not exist static ProjectDiff localProjectChanges( const QString &projectDir ); static bool hasLocalProjectChanges( const QString &projectDir, bool supportsSelectiveSync ); @@ -437,7 +447,7 @@ class MerginApi: public QObject * \param minor parsed minor number * @return true when parsing was successful */ - static bool parseVersion( const QString &version, int &major, int &minor ); + static bool parseVersion( const QString &version, int &major, int &minor ); // todo: go to utils /** * Parse major, minor and patch version number from version string. @@ -447,14 +457,14 @@ class MerginApi: public QObject * \param patch parsed patch number * @return true when parsing was successful */ - static bool parseVersion( const QString &version, int &major, int &minor, int &patch ); + static bool parseVersion( const QString &version, int &major, int &minor, int &patch ); // todo: go to utils /** * Finds project in merginProjects list according its full name. * \param projectPath Full path to project's folder * \param metadataFile Relative path of metafile to project's folder */ - Q_INVOKABLE static QString getFullProjectName( QString projectNamespace, QString projectName ); + Q_INVOKABLE static QString getFullProjectName( QString projectNamespace, QString projectName ); // todo: go to utils, drop invokable /** * Creates an empty project on Mergin server. isPublic determines if the new project will be visible to all or private @@ -542,7 +552,7 @@ class MerginApi: public QObject } // Returns true for files that are under .mergin folder or contains ignored extension from sIgnoreExtensions - static bool isInIgnore( const QFileInfo &info ); + static bool isInIgnore( const QFileInfo &info ); // todo: go to utils /** * Performs checks and returns if a given file is excluded from the sync. @@ -566,7 +576,7 @@ class MerginApi: public QObject * \param projectNamespace QString to be set as namespace, might not change original value * \param projectName QString to be set to name of a project */ - static bool extractProjectName( const QString &sourceString, QString &projectNamespace, QString &projectName ); + static bool extractProjectName( const QString &sourceString, QString &projectNamespace, QString &projectName ); // todo: go to utils bool supportsSelectiveSync() const; void setSupportsSelectiveSync( bool supportsSelectiveSync ); @@ -589,7 +599,7 @@ class MerginApi: public QObject /** * Reads server details and user details from QSettings. */ - void loadCache(); + void loadCache(); // Should mergin api has any idea of offline cache? /** * Sends non-blocking GET request to the server to list user workspaces. @@ -628,7 +638,7 @@ class MerginApi: public QObject /** * Emits API calls that bear user information like username, workspaces and service */ - Q_INVOKABLE void refreshUserData(); + Q_INVOKABLE void refreshUserData(); //? /** * Returns true if server supports workspaces @@ -788,8 +798,10 @@ class MerginApi: public QObject // Push slots void pushStartReplyFinished(); + void pushStartV2ReplyFinished(); void pushInfoReplyFinished(); void pushFileReplyFinished(); + void pushV2FileReplyFinished(); void pushFinishReplyFinished(); void pushCancelReplyFinished(); @@ -830,20 +842,19 @@ class MerginApi: public QObject void abortPullItems( const QString &projectFullName ); /** - * Sends non-blocking POST request to the server to upload a file (chunk). + * Sends non-blocking POST request to the server to initiate project push. * \param projectFullName Namespace/name - * \param json project info containing metadata for upload */ - void pushStart( const QString &projectFullName, const QByteArray &json ); + void pushStart( const QString &projectFullName ); + void preparePushPayload( const QString &projectFullName ); /** * Sends non-blocking POST request to the server to upload a file (chunk). * \param projectFullName Namespace/name - * \param transactionUUID Transaction ID which servers sends on uploadStart * \param file Mergin file to upload * \param chunkNo Chunk number of given file to be uploaded */ - void pushFile( const QString &projectFullName, const QString &transactionUUID, MerginFile file, int chunkNo = 0 ); + void pushFile( const QString &projectFullName, MerginFile file, int chunkNo = 0 ); /** * Closing request after successful push. @@ -851,6 +862,7 @@ class MerginApi: public QObject * \param transactionUUID transaction UUID to match upload process on the server */ void pushFinish( const QString &projectFullName, const QString &transactionUUID ); + void pushV2Finish( const QString &projectFullName ); void sendPushCancelRequest( const QString &projectFullName, const QString &transactionUUID ); @@ -954,7 +966,9 @@ class MerginApi: public QObject AttrProjectFullName = QNetworkRequest::User, AttrTempFileName = QNetworkRequest::User + 1, AttrWorkspaceName = QNetworkRequest::User + 2, - AttrAcceptFlag = QNetworkRequest::User + 3 + AttrAcceptFlag = QNetworkRequest::User + 3, + AttrChunkNo = QNetworkRequest::User + 4, + AttrChunkSize = QNetworkRequest::User + 5 }; Transactions mTransactionalStatus; //projectFullname -> transactionStatus diff --git a/core/merginprojectmetadata.cpp b/core/merginprojectmetadata.cpp index 7b4f2610e..b78064a87 100644 --- a/core/merginprojectmetadata.cpp +++ b/core/merginprojectmetadata.cpp @@ -97,8 +97,22 @@ MerginProjectMetadata MerginProjectMetadata::fromJson( const QByteArray &data ) project.files << MerginFile::fromJsonObject( it->toObject() ); } + project.id = docObj.value( QStringLiteral( "id" ) ).toString(); project.name = docObj.value( QStringLiteral( "name" ) ).toString(); - project.projectNamespace = docObj.value( QStringLiteral( "namespace" ) ).toString(); + + if ( docObj.contains( QStringLiteral( "workspace" ) ) ) + { + // v2 project detail + QJsonObject workspaceData = docObj.value( QStringLiteral( "workspace" ) ).toObject(); + + project.workspaceId = workspaceData.value( QStringLiteral( "id" ) ).toInt(); + project.workspaceName = workspaceData.value( QStringLiteral( "name" ) ).toString(); + } + else + { + project.workspaceName = docObj.value( QStringLiteral( "namespace" ) ).toString(); + } + project.role = docObj.value( QStringLiteral( "role" ) ).toString(); QString versionStr = docObj.value( QStringLiteral( "version" ) ).toString(); @@ -112,15 +126,6 @@ MerginProjectMetadata MerginProjectMetadata::fromJson( const QByteArray &data ) project.version = versionStr.toInt(); } - if ( docObj.contains( QStringLiteral( "id" ) ) ) - { - project.projectId = docObj.value( QStringLiteral( "id" ) ).toString(); - } - else - { - project.projectId.clear(); - } - return project; } diff --git a/core/merginprojectmetadata.h b/core/merginprojectmetadata.h index 00705add7..b2e60fb97 100644 --- a/core/merginprojectmetadata.h +++ b/core/merginprojectmetadata.h @@ -14,13 +14,21 @@ #include #include +struct FileChunkV2 +{ + // struct used in push v2 when server responds after chunk upload + QString id; // chunk id, as generated on server + QDateTime valid_until; // chunk can be reused and not reuploaded until this datetime +}; + struct MerginFile { QString path; QString checksum; qint64 size; QDateTime mtime; - QStringList chunks; // used only for upload otherwise suppose to be empty + + QStringList chunks; // pregenerated chunks to upload; uuid in v1, temp id in v2 that is filled by server value after chunk is uploaded // // these are members only used for upload of changed file through a geo-diff @@ -57,16 +65,18 @@ struct MerginConfig //! Metadata read from project info reply or read from cached local .mergin.json file struct MerginProjectMetadata { + QString id; //!< unique project ID (only available in API that supports project IDs) QString name; - QString projectNamespace; + int workspaceId = -1; //! empty when using v1 project detail + QString workspaceName; + QString role; int version = -1; QList files; - QString projectId; //!< unique project ID (only available in API that supports project IDs) // no project dir, no sync state, ... - bool isValid() const { return !name.isEmpty() && !projectNamespace.isEmpty(); } + bool isValid() const { return !name.isEmpty() && !workspaceName.isEmpty(); } static MerginProjectMetadata fromJson( const QByteArray &data );