Encryption settings in API for backups to fs (#28305)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: stanislav-shchetinin

ydb/core/grpc_services/rpc_import.cpp

@@ -131,6 +131,11 @@ class TImportRPC: public TRpcOperationRequestActor<TDerived, TEvRequest, true>,
                 return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, 
                     "base_path must be an absolute path");
             }
+            for (const auto& item : settings.items()) {
+                if (item.destination_path().empty() && item.source_path().empty()) {
+                    return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Empty item is not allowed");
+                }
+            }
         }
 
         this->AllocateTxId();

ydb/public/api/grpc/ydb_export_v1.proto

@@ -17,5 +17,5 @@ service ExportService {
 
     // Exports data to file system.
     // Method starts an asynchronous operation that can be cancelled while it is in progress.
-    rpc ExportToFs(ExportToFsRequest) returns (ExportToFsResponse);
+    rpc ExportToFs(Export.ExportToFsRequest) returns (Export.ExportToFsResponse);
 }

ydb/public/api/grpc/ydb_import_v1.proto

@@ -18,6 +18,9 @@ service ImportService {
     // List objects from existing export stored in S3 bucket
     rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse);
 
+    // List objects from existing export stored in FS
+    rpc ListObjectsInFsExport(Import.ListObjectsInFsExportRequest) returns (Import.ListObjectsInFsExportResponse);
+
     // Writes data to a table.
     // Method accepts serialized data in the selected format and writes it non-transactionally.
     rpc ImportData(Import.ImportDataRequest) returns (Import.ImportDataResponse);

ydb/public/api/protos/ydb_export.proto

@@ -126,7 +126,7 @@ message ExportToS3Settings {
     // it is especially useful for custom s3 implementations
     bool disable_virtual_addressing = 12;
 
-    // Database root if not provided.
+    // Defaults to database root if not provided.
     // All object names are calculated and written relative to this path.
     string source_path = 13;
 
@@ -185,18 +185,23 @@ message EncryptionSettings {
 /// File system (FS)
 message ExportToFsSettings {
     message Item {
-        // Database path to a table to be exported
+        // Database path to a table/directory to be exported
         string source_path = 1 [(required) = true];
 
-        /* The tables are exported to one or more files in FS.
-        The files are saved to the destination_path directory. */
-        string destination_path = 2 [(required) = true];
+        /* Tables are exported to one or more files in FS.
+           The path begins with 'destination_path'.
+           If not specified, actual FS path is the default `base_path` concatenated with:
+            * The object path relative to the global `source_path` for a non-encrypted export
+            * The anonymized path for an encrypted export
+        */
+        string destination_path = 2;
     }
 
-    // Base path on FS where to write export
-    // Path to the mounted directory in the case of NFS
+    // Base path on FS where to write all export items
+    // In the case of NFS, one of the directories in the path must be mounted
     // Must be an absolute path
     // Example: /mnt/exports
+    // SchemaMapping file with the list of objects is written to this path
     string base_path = 1 [(required) = true];
 
     // List of items to export
@@ -212,6 +217,15 @@ message ExportToFsSettings {
     // - zstd.
     // - zstd-N, where N is compression level, e.g. zstd-3.
     string compression = 5;
+
+    // Settings for data encryption.
+    // If encryption_settings field is not specified,
+    // the resulting data will not be encrypted.
+    EncryptionSettings encryption_settings = 6;
+
+    // Defaults to database root if not provided.
+    // All object names are calculated and written relative to this path.
+    string source_path = 7;
 }
 
 message ExportToFsResult {

ydb/public/api/protos/ydb_import.proto

@@ -131,19 +131,23 @@ message ImportFromFsSettings {
              * '/scheme.pb' - object with information about scheme, indexes, etc;
              * '/permissions.pb' - object with information about ACL and owner;
              * '/metadata.json' - object with metadata about the backup.
-           The path in FS is specified relative to base_path.
-           Example: "my_export/table1"
+        The FS path can be either provided explicitly (relative to base_path)
+        Or, if the export contains the database objects list, you may specify the database object name,
+        and the FS prefix will be looked up in the database objects list by the import procedure
         */
-        string source_path = 1 [(required) = true];
+        string source_path = 1;
 
-        // Database path to a table to import to.
-        string destination_path = 2 [(required) = true];
+        // Database path to a database object to import the item to
+        // Resolved relative to the default destination_path
+        // May be omitted if the item's source_path is specified, in this case will be taken equal to it
+        string destination_path = 2;
     }
 
     // Base path on FS where the export is located
-    // Path to the mounted directory in the case of NFS
+    // In the case of NFS, one of the directories in the path must be mounted
     // Must be an absolute path
     // Example: /mnt/exports
+    // SchemaMapping file with the list of objects is read from this path
     string base_path = 1 [(required) = true];
 
     repeated Item items = 2; // Empty collection means import of all export objects
@@ -160,6 +164,15 @@ message ImportFromFsSettings {
 
     // Skip checksum validation during import
     bool skip_checksum_validation = 6;
+
+    // Destination path to restore paths inside database
+    // Default value is database root
+    string destination_path = 7;
+
+    // Settings how data is encrypted.
+    // If encryption_settings field is not specified,
+    // the resulting data is considered not encrypted.
+    Ydb.Export.EncryptionSettings encryption_settings = 8;
 }
 
 message ImportFromFsResult {
@@ -263,6 +276,66 @@ message ListObjectsInS3ExportResponse {
     Ydb.Operations.Operation operation = 1;
 }
 
+message ListObjectsInFsExportSettings {
+    message Item {
+        // Database object path
+        // Recursive for directories
+        string path = 1;
+    }
+
+    string base_path = 1 [(required) = true];
+    repeated Item items = 2;
+    uint32 number_of_retries = 3;
+
+    // Settings how data is encrypted.
+    // If encryption_settings field is not specified,
+    // the resulting data is considered not encrypted.
+    Ydb.Export.EncryptionSettings encryption_settings = 4;
+}
+
+message ListObjectsInFsExportResult {
+    message Item {
+        /* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto).
+          The S3 object name begins with a prefix, followed by:
+            * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
+            * '/scheme.pb' - object with information about scheme, indexes, etc;
+            * '/permissions.pb' - object with information about ACL and owner;
+            * '/metadata.json' - object with metadata about the backup.
+        */
+        string fs_path = 1;
+        string db_path = 2;
+    }
+
+    repeated Item items = 1;
+
+    // This token allows you to get the next page of results for ListObjectsInFsExport requests,
+    // if the number of results is larger than `page_size` specified in the request.
+    // To get the next page, specify the value of `next_page_token` as a value for
+    // the `page_token` parameter in the next ListObjectsInFsExport request. Subsequent ListObjectsInFsExport
+    // requests will have their own `next_page_token` to continue paging through the results.
+    string next_page_token = 2;
+}
+
+message ListObjectsInFsExportRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    ListObjectsInFsExportSettings settings = 2 [(required) = true];
+
+    // The maximum number of results per page that should be returned. If the number of available
+    // results is larger than `page_size`, the service returns a `next_page_token` that can be used
+    // to get the next page of results in subsequent ListObjectsInFsExport requests.
+    // 0 means that server returns all objects.
+    int64 page_size = 3 [(value) = "<= 10000"];
+
+    // Page token. Set `page_token` to the `next_page_token` returned by a previous ListObjectsInFsExport
+    // request to get the next page of results.
+    string page_token = 4;
+}
+
+message ListObjectsInFsExportResponse {
+    // operation.result = ListObjectsInFsExportResult
+    Ydb.Operations.Operation operation = 1;
+}
+
 /// Data
 message YdbDumpFormat {
     repeated string columns = 1;

ydb/services/ydb/backup_ut/fs_backup_validation_ut.cpp

@@ -292,7 +292,7 @@ Y_UNIT_TEST_SUITE_F(FsImportParamsValidationTest, TFsBackupParamsValidationTestF
         UNIT_ASSERT_VALUES_EQUAL_C(res.Status().GetStatus(), NYdb::EStatus::BAD_REQUEST, 
             res.Status().GetIssues().ToString());
         UNIT_ASSERT_STRING_CONTAINS_C(res.Status().GetIssues().ToString(), 
-            "source_path is required but not set",
+            "Empty item is not allowed",
             res.Status().GetIssues().ToString());
     }
 }