Merge pull request #1466 from 0chain/feat/file-search

Add file search using names

Author: dabasov

code/go/0chain.net/blobbercore/allocation/newdirchange.go

@@ -19,6 +19,7 @@ type NewDir struct {
 	ConnectionID string `json:"connection_id" validation:"required"`
 	Path         string `json:"filepath" validation:"required"`
 	AllocationID string `json:"allocation_id"`
+	CustomMeta   string `json:"custom_meta,omitempty"`
 }
 
 func (nf *NewDir) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
@@ -79,6 +80,7 @@ func (nf *NewDir) ApplyChange(ctx context.Context, rootRef *reference.Ref, chang
 					fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path))
 			}
 			newRef.FileID = fileID
+			newRef.CustomMeta = nf.CustomMeta
 			dirRef.AddChild(newRef)
 			dirRef = newRef
 		}

code/go/0chain.net/blobbercore/handler/health.go

@@ -52,6 +52,8 @@ func BlobberHealthCheck() (*transaction.Transaction, error) {
 		return nil, err
 	}
 
+	setBlobberHealthCheckError(nil)
+
 	return txn, nil
 }
 

code/go/0chain.net/blobbercore/handler/object_operation_handler.go

@@ -656,11 +656,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
 		return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation")
 	}
 
-	if allocationObj.BlobberSizeUsed+connectionObj.Size > allocationObj.BlobberSize {
-		return nil, common.NewError("max_allocation_size",
-			"Max size reached for the allocation with this blobber")
-	}
-
 	writeMarkerString := r.FormValue("write_marker")
 	if writeMarkerString == "" {
 		return nil, common.NewError("invalid_parameters", "Invalid write marker passed")
@@ -689,11 +684,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
 				"Latest write marker is in failed state")
 		}
 
-		if latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size != writeMarker.ChainSize {
-			return nil, common.NewErrorf("invalid_chain_size",
-				"Invalid chain size. expected:%v got %v", latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size, writeMarker.ChainSize)
-		}
-
 		if latestWriteMarkerEntity.Status != writemarker.Committed {
 			writeMarker.ChainLength = latestWriteMarkerEntity.WM.ChainLength
 		}
@@ -758,6 +748,24 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
 	if !rootRef.IsPrecommit {
 		return nil, common.NewError("no_root_change", "No change in root ref")
 	}
+	connectionObj.Size = rootRef.Size - allocationObj.BlobberSizeUsed
+
+	if writemarkerEntity.WM.Size != connectionObj.Size {
+		return nil, common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker size %v does not match the connection size %v", writemarkerEntity.WM.Size, connectionObj.Size))
+	}
+
+	if allocationObj.BlobberSizeUsed+connectionObj.Size > allocationObj.BlobberSize {
+		return nil, common.NewError("max_allocation_size",
+			"Max size reached for the allocation with this blobber")
+	}
+
+	if latestWriteMarkerEntity != nil && latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size != writeMarker.ChainSize {
+		return nil, common.NewErrorf("invalid_chain_size",
+			"Invalid chain size. expected:%v got %v", latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size, writeMarker.ChainSize)
+	} else if latestWriteMarkerEntity == nil && connectionObj.Size != writeMarker.ChainSize {
+		return nil, common.NewErrorf("invalid_chain_size",
+			"Invalid chain size. expected:%v got %v", connectionObj.Size, writeMarker.ChainSize)
+	}
 
 	elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock -
 		elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem
@@ -1245,6 +1253,8 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*all
 		return nil, common.NewError("invalid_parameters", "Invalid dir path passed")
 	}
 
+	customMeta := r.FormValue("custom_meta")
+
 	exisitingRef, err := fsh.checkIfFileAlreadyExists(ctx, allocationID, dirPath)
 	if err != nil {
 		Logger.Error("Error file reference", zap.Error(err))
@@ -1257,6 +1267,16 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*all
 	if exisitingRef != nil {
 		// target directory exists, return StatusOK
 		if exisitingRef.Type == reference.DIRECTORY {
+
+			if exisitingRef.CustomMeta != customMeta {
+				_ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+					err := reference.UpdateCustomMeta(ctx, exisitingRef, customMeta)
+					if err != nil {
+						logging.Logger.Error("Error updating custom meta", zap.Error(err))
+					}
+					return err
+				})
+			}
 			return nil, common.NewError("directory_exists", "Directory already exists`")
 		}
 
@@ -1293,6 +1313,7 @@ func (fsh *StorageHandler) CreateDir(ctx context.Context, r *http.Request) (*all
 	newDir.ConnectionID = connectionID
 	newDir.Path = dirPath
 	newDir.AllocationID = allocationID
+	newDir.CustomMeta = customMeta
 
 	connectionObj.AddChange(allocationChange, &newDir)
 

code/go/0chain.net/blobbercore/handler/storage_handler.go

@@ -103,7 +103,7 @@ func (fsh *StorageHandler) GetAllocationUpdateTicket(ctx context.Context, r *htt
 }
 
 func (fsh *StorageHandler) checkIfFileAlreadyExists(ctx context.Context, allocationID, path string) (*reference.Ref, error) {
-	return reference.GetLimitedRefFieldsByPath(ctx, allocationID, path, []string{"id", "type"})
+	return reference.GetLimitedRefFieldsByPath(ctx, allocationID, path, []string{"id", "type", "custom_meta"})
 }
 
 func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (interface{}, error) {
@@ -276,7 +276,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i
 
 // swagger:route GET /v1/file/list/{allocation} GetListFiles
 // List files.
-// ListHandler is the handler to respond to list requests from clients, 
+// ListHandler is the handler to respond to list requests from clients,
 // it returns a list of files in the allocation,
 // along with the metadata of the files.
 //
@@ -344,7 +344,7 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i
 // responses:
 //
 //   200: ListResult
-//   400: 
+//   400:
 
 func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*blobberhttp.ListResult, error) {
 	clientID := ctx.Value(constants.ContextKeyClient).(string)

code/go/0chain.net/blobbercore/reference/ref.go

@@ -60,7 +60,7 @@ type Ref struct {
 	PathHash                string `gorm:"column:path_hash;size:64;not null" dirlist:"path_hash" filelist:"path_hash"`
 	ParentPath              string `gorm:"column:parent_path;size:999;index:idx_parent_path_alloc,priority:2"`
 	PathLevel               int    `gorm:"column:level;not null;default:0"`
-	CustomMeta              string `gorm:"column:custom_meta;not null" filelist:"custom_meta"`
+	CustomMeta              string `gorm:"column:custom_meta;not null" filelist:"custom_meta" dirlist:"custom_meta"`
 	ValidationRoot          string `gorm:"column:validation_root;size:64;not null;index:idx_validation_alloc,priority:2" filelist:"validation_root"`
 	PrevValidationRoot      string `gorm:"column:prev_validation_root" filelist:"prev_validation_root" json:"prev_validation_root"`
 	ValidationRootSignature string `gorm:"column:validation_root_signature;size:64" filelist:"validation_root_signature" json:"validation_root_signature,omitempty"`
@@ -144,6 +144,7 @@ type PaginatedRef struct { //Gorm smart select fields.
 	ActualThumbnailHash     string `gorm:"column:actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"`
 	EncryptedKey            string `gorm:"column:encrypted_key" json:"encrypted_key,omitempty"`
 	EncryptedKeyPoint       string `gorm:"column:encrypted_key_point" json:"encrypted_key_point,omitempty"`
+	FileMetaHash            string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"`
 
 	CreatedAt common.Timestamp `gorm:"column:created_at" json:"created_at,omitempty"`
 	UpdatedAt common.Timestamp `gorm:"column:updated_at" json:"updated_at,omitempty"`
@@ -282,8 +283,8 @@ func GetReferenceByLookupHashForDownload(ctx context.Context, allocationID, path
 func GetReferencesByName(ctx context.Context, allocationID, name string) (refs []*Ref, err error) {
 	db := datastore.GetStore().GetTransaction(ctx)
 	err = db.Model(&Ref{}).
-		Where("allocation_id = ? AND name @@ plainto_tsquery(?)", allocationID, name).
-		Order("ts_rank_cd(to_tsvector('english', name), plainto_tsquery(?)) DESC").
+		Where("allocation_id = ? AND name LIKE ?", allocationID, "%"+name+"%").
+		Limit(20).
 		Find(&refs).Error
 	if err != nil {
 		return nil, err
@@ -669,3 +670,8 @@ func GetListingFieldsMap(refEntity interface{}, tagName string) map[string]inter
 	}
 	return result
 }
+
+func UpdateCustomMeta(ctx context.Context, ref *Ref, customMeta string) error {
+	db := datastore.GetStore().GetTransaction(ctx)
+	return db.Exec("UPDATE reference_objects SET custom_meta = ? WHERE id = ?", customMeta, ref.ID).Error
+}

code/go/0chain.net/blobbercore/writemarker/protocol.go

@@ -76,10 +76,6 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al
 		return common.NewError("write_marker_validation_failed", "Write Marker is not for the same allocation transaction")
 	}
 
-	if wme.WM.Size != co.Size {
-		return common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker size %v does not match the connection size %v", wme.WM.Size, co.Size))
-	}
-
 	clientPublicKey := ctx.Value(constants.ContextKeyClientKey).(string)
 	if clientPublicKey == "" {
 		return common.NewError("write_marker_validation_failed", "Could not get the public key of the client")