diff --git a/backend/internal/app/runtime_plugin.go b/backend/internal/app/runtime_plugin.go index c881166a2..a0f361420 100644 --- a/backend/internal/app/runtime_plugin.go +++ b/backend/internal/app/runtime_plugin.go @@ -58,10 +58,24 @@ func providePluginSource(cfg *config.Config, store *postgres.Store, configuratio // providePluginManager assembles verified storage, installation policy, and process ownership. // The source is shared with host capabilities to avoid a manager/process dependency // cycle. Construction leaves reconciliation to the registered application lifecycle hook. -func providePluginManager(cfg *config.Config, info BuildInfo, source *plugin.DevSource, configuration *plugin.Configuration, packages *provider.PackageFiles, processes *provider.Processes) *plugin.Manager { +func providePluginManager( + cfg *config.Config, + info BuildInfo, + source *plugin.DevSource, + configuration *plugin.Configuration, + packages *provider.PackageFiles, + processes *provider.Processes, +) (*plugin.Manager, error) { + data, err := provider.NewDataFiles(filepath.Join(cfg.Plugins.Dir, "data")) + if err != nil { + return nil, err + } return plugin.NewManager(plugin.ManagerOptions{ - Enabled: cfg.Plugins.Enabled, MaxRunning: cfg.Plugins.MaxRunning, HostVersion: info.Version, StateRoot: filepath.Join(cfg.Plugins.Dir, "state"), - }, source, packages, processes, configuration, slog.Default()) + Enabled: cfg.Plugins.Enabled, + MaxRunning: cfg.Plugins.MaxRunning, + HostVersion: info.Version, + StateRoot: filepath.Join(cfg.Plugins.Dir, "state"), + }, source, packages, processes, configuration, data, slog.Default()), nil } // providePluginService binds administration to the same runtime manager and staged uploads. diff --git a/backend/internal/app/testdata/routes.json b/backend/internal/app/testdata/routes.json index 0fa6cdb9b..524119cf7 100644 --- a/backend/internal/app/testdata/routes.json +++ b/backend/internal/app/testdata/routes.json @@ -15,6 +15,7 @@ "DELETE /api/v1/admin/payment/providers/:id", "DELETE /api/v1/admin/plugin-indexes/:id", "DELETE /api/v1/admin/plugins/:id", + "DELETE /api/v1/admin/plugins/:id/data", "DELETE /api/v1/admin/pricing/configs/:id", "DELETE /api/v1/admin/promo-codes/:id", "DELETE /api/v1/admin/providers/:id", @@ -156,8 +157,10 @@ "GET /api/v1/admin/plugins", "GET /api/v1/admin/plugins/:id", "GET /api/v1/admin/plugins/:id/config", + "GET /api/v1/admin/plugins/:id/data/archive", "GET /api/v1/admin/plugins/:id/limits", "GET /api/v1/admin/plugins/:id/logs", + "GET /api/v1/admin/plugins/data", "GET /api/v1/admin/pricing/configs", "GET /api/v1/admin/pricing/configs/:id", "GET /api/v1/admin/pricing/defaults", @@ -442,6 +445,7 @@ "POST /api/v1/admin/plugin-catalog/refresh", "POST /api/v1/admin/plugin-indexes", "POST /api/v1/admin/plugins", + "POST /api/v1/admin/plugins/:id/data/archive", "POST /api/v1/admin/plugins/:id/disable", "POST /api/v1/admin/plugins/:id/enable", "POST /api/v1/admin/plugins/:id/restart", diff --git a/backend/internal/app/wire_gen.go b/backend/internal/app/wire_gen.go index 51e642fe6..fc5d44f73 100644 --- a/backend/internal/app/wire_gen.go +++ b/backend/internal/app/wire_gen.go @@ -471,7 +471,10 @@ func initializeApplication(ctx context.Context, cfg *config.Config, info BuildIn redeemCodes := providePluginRedeemCodes(writes, grants, plans, redeemService) hostServices := providePluginHostServices(walletService, subscriptions, queries, emails, captchaVerifier, kv, redeemCodes, grants, auditLogService) processes := providePluginProcesses(cfg, info, hostServices) - pluginManager := providePluginManager(cfg, info, devSource, configuration, packageFiles, processes) + pluginManager, err := providePluginManager(cfg, info, devSource, configuration, packageFiles, processes) + if err != nil { + return nil, err + } catalogService, err := providePluginCatalog(cfg, info, store2, devSource) if err != nil { return nil, err diff --git a/backend/internal/pkg/locale/error_messages.json b/backend/internal/pkg/locale/error_messages.json index dec5e9ac8..3deb90d2c 100644 --- a/backend/internal/pkg/locale/error_messages.json +++ b/backend/internal/pkg/locale/error_messages.json @@ -180,7 +180,12 @@ "MAX_RUNNING_EXCEEDED": "Plugin process limit reached", "PLUGIN_INDEX_SIGNATURE_INVALID": "Plugin index signature is invalid", "PLUGIN_CONFIG_INVALID": "Plugin configuration is invalid", - "PLUGIN_PACKAGE_MISSING": "Plugin package is missing; upload the same version again" + "PLUGIN_PACKAGE_MISSING": "Plugin package is missing; upload the same version again", + "PLUGIN_DATA_REQUIRES_STOP": "Disable the plugin before backing up or restoring its data.", + "PLUGIN_DATA_NOT_FOUND": "Plugin data not found.", + "PLUGIN_DATA_EXISTS": "Plugin data directory is not empty.", + "PLUGIN_DATA_INVALID": "Plugin data archive or directory is invalid.", + "PLUGIN_DATA_TOO_LARGE": "Plugin data exceeds archive limits." }, "zh-Hans": { "HTTP_400": "请求参数无效，请检查后重试。", @@ -363,6 +368,11 @@ "MAX_RUNNING_EXCEEDED": "已达到插件进程数量上限。", "PLUGIN_INDEX_SIGNATURE_INVALID": "插件索引签名无效。", "PLUGIN_CONFIG_INVALID": "插件配置无效，请检查字段。", - "PLUGIN_PACKAGE_MISSING": "插件安装包丢失，请重新上传相同版本。" + "PLUGIN_PACKAGE_MISSING": "插件安装包丢失，请重新上传相同版本。", + "PLUGIN_DATA_REQUIRES_STOP": "备份或恢复数据前请先停用插件。", + "PLUGIN_DATA_NOT_FOUND": "插件数据不存在。", + "PLUGIN_DATA_EXISTS": "插件数据目录非空，请先备份并清理现有数据。", + "PLUGIN_DATA_INVALID": "插件数据备份或目录无效。", + "PLUGIN_DATA_TOO_LARGE": "插件数据超过备份大小或条目数量限制。" } } diff --git a/backend/internal/plugin/data.go b/backend/internal/plugin/data.go new file mode 100644 index 000000000..f755ee4c9 --- /dev/null +++ b/backend/internal/plugin/data.go @@ -0,0 +1,169 @@ +package plugin + +import ( + "context" + "errors" + "io" +) + +// data.go coordinates durable plugin directories with installation ownership. +// Files survive ordinary uninstall and remain discoverable by their plugin ID. +// Backup and restore require disabled or uninstalled plugins, and use the same +// per-plugin lock as installation changes so execution cannot restart midway. +// +// The file store owns directory validation and archive integrity. Management +// owns admission and callback draining before accessing stopped business data. +// Database KV and local files have separate commits: a failed file purge remains +// discoverable and can be retried after the installation row has been removed. +// External databases and object stores retain their operator-managed lifecycle. + +const ( + // DataMaxArchiveBytes bounds temporary disk use for compressed archives and expanded business files. + DataMaxArchiveBytes = int64(1 << 30) +) + +// DataUsage describes local business files without exposing their absolute paths. +type DataUsage struct { + PluginID string `json:"plugin_id"` // publisher-qualified data owner + Bytes int64 `json:"bytes"` // regular file bytes + Files int64 `json:"files"` // regular file count + // Installed reports record presence; Enabled follows that record's desired + // execution state and stays false when the installation has been removed, even + // while local business files remain available for reinstallation. + Installed bool `json:"installed"` // installation record currently exists + Enabled bool `json:"enabled"` // installation currently requests execution +} + +// DataArchive owns a completed immutable backup and its temporary file cleanup. +type DataArchive struct { + // Reader owns the immutable snapshot and its temporary file. Callers close + // it after delivery, including interrupted responses, to release that storage. + Reader io.ReadCloser + Bytes int64 // compressed archive byte length +} + +// DataStorage manages plugin-owned directories and validated offline file archives. +// @project-doc docs/architecture/plugin_system.md#persistent_data +type DataStorage interface { + // Directory creates the stable directory for id and returns its absolute path. + // Callers serialize creation with purge and restore for this plugin. Existing + // data remains available across package replacements and ordinary uninstallation. + Directory(context.Context, string) (string, error) + // List measures retained directories without following links into other owners. + List(context.Context) ([]DataUsage, error) + // Remove deletes one owner's files after callers drain all plugin work. + // Missing directories succeed, allowing cleanup to resume after partial failure. + // Retained host financial and notification ledgers belong to their existing stores. + Remove(context.Context, string) error + // Backup snapshots stopped business files into a checksummed portable archive. + // Callers hold the plugin management lock and close the returned reader after + // use, releasing its temporary storage even when response delivery fails. + Backup(context.Context, string) (*DataArchive, error) + // Restore validates a complete archive before publishing an absent or empty directory. + // Nonempty destinations are rejected. Callers keep the plugin stopped and hold + // its management lock throughout validation and publication of restored files. + Restore(context.Context, string, io.Reader) error +} + +// ListData returns durable directory usage including uninstalled plugin owners. +// Sizes are observations rather than quotas; running plugins may change them. +// Each record is annotated from current installation intent without reading secrets. +func (s *Service) ListData(ctx context.Context) ([]DataUsage, error) { + items, err := s.manager.data.List(ctx) + if err != nil { + return nil, err + } + installed, err := s.manager.store.List(ctx) + if err != nil { + return nil, err + } + byID := make(map[string]Installation, len(installed)) + for _, item := range installed { + byID[item.PluginID] = item + } + for i := range items { + item, ok := byID[items[i].PluginID] + items[i].Installed = ok + items[i].Enabled = ok && item.Enabled + } + return items, nil +} + +// PurgeData clears local files and KV for an uninstalled plugin identity. +// An installed owner must be removed first. The management lock prevents a +// concurrent reinstall from acquiring the directory while deletion is in progress. +// +// Host KV commits before local files are removed, because those stores cannot +// share a transaction. A filesystem failure leaves the owner discoverable for +// retry; successful cleanup also removes any package state left behind by an +// earlier partially completed uninstall. Financial and delivery ledger history remains. +func (s *Service) PurgeData(ctx context.Context, id string) error { + return s.manager.locked(ctx, id, func() error { + _, err := s.manager.store.Get(ctx, id) + if err == nil { + return ErrPluginAlreadyInstalled + } + if !errors.Is(err, ErrPluginNotFound) { + return err + } + if err := s.manager.drainLocked(ctx, id); err != nil { + return err + } + // Delete host KV before filesystem cleanup so failed file removal remains + // visible and retryable without allowing a new executable generation between steps. + if err := s.manager.store.Delete(ctx, id, true); err != nil { + return err + } + if err := s.manager.data.Remove(ctx, id); err != nil { + return err + } + return s.manager.syncLocked(ctx, id) + }) +} + +// BackupData produces a stable snapshot of a disabled or uninstalled owner's files. +// The returned archive is independent of later execution and belongs to the caller. +// Installed configuration, host KV, ledgers, and external stores require their respective backups. +// +// Stopped intent is checked while holding the same owner lock used by enable and +// reinstall, then actual process and callback completion is awaited. Only snapshot +// creation holds that lock; delivery reads an immutable temporary file, allowing +// later execution without making the downloaded bytes inconsistent. +func (s *Service) BackupData(ctx context.Context, id string) (*DataArchive, error) { + var archive *DataArchive + err := s.manager.locked(ctx, id, func() error { + if err := s.dataStopped(ctx, id); err != nil { + return err + } + var err error + archive, err = s.manager.data.Backup(ctx, id) + return err + }) + return archive, err +} + +// RestoreData publishes validated business files while keeping their owner stopped. +// The archive must match id and the destination must be absent or empty. Current +// installation configuration and permissions remain governed by ordinary administration. +func (s *Service) RestoreData(ctx context.Context, id string, reader io.Reader) error { + return s.manager.locked(ctx, id, func() error { + if err := s.dataStopped(ctx, id); err != nil { + return err + } + return s.manager.data.Restore(ctx, id, reader) + }) +} + +// dataStopped checks disabled intent and waits for actual executable callback completion. +// It runs under the owner's management lock. Drain failure leaves files untouched, +// and an enabled installation requires an administrator to disable it before retrying. +func (s *Service) dataStopped(ctx context.Context, id string) error { + item, err := s.manager.store.Get(ctx, id) + if err != nil && !errors.Is(err, ErrPluginNotFound) { + return err + } + if item != nil && item.Enabled { + return ErrDataRequiresStop + } + return s.manager.drainLocked(ctx, id) +} diff --git a/backend/internal/plugin/data_test.go b/backend/internal/plugin/data_test.go new file mode 100644 index 000000000..a0edbef84 --- /dev/null +++ b/backend/internal/plugin/data_test.go @@ -0,0 +1,97 @@ +package plugin + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// data_test.go checks persistent file management against installation and process state. +// Storage callbacks are controlled independently from host KV, exposing ordering, +// retry, and stopped-owner decisions without starting an executable or external database. + +// dataFixture supplies the management observations needed by persistent data tests. +type dataFixture struct { + DataStorage // methods unused by individual scenarios + removed []string // local owner purge attempts + removeError error // injected filesystem cleanup failure + backups int // accepted offline snapshot attempts + path string // persistent directory supplied to processes +} + +// Directory returns the fixture's stable path for native process initialization. +func (f *dataFixture) Directory(context.Context, string) (string, error) { + return f.path, nil +} + +// Remove records each cleanup attempt and returns the configured filesystem outcome. +func (f *dataFixture) Remove(_ context.Context, id string) error { + f.removed = append(f.removed, id) + return f.removeError +} + +// Backup counts snapshots admitted after management has checked stopped-owner policy. +func (f *dataFixture) Backup(context.Context, string) (*DataArchive, error) { + f.backups++ + return &DataArchive{Reader: io.NopCloser(strings.NewReader(""))}, nil +} + +// TestDataDirectoryReachesNativeProcess starts one accepted native contribution. +// Its immutable process specification receives the durable path separately from disposable state, and ordinary uninstall retains it. +func TestDataDirectoryReachesNativeProcess(t *testing.T) { + manager, service, store, processes, _ := managerFixture(t, 1) + files := &dataFixture{path: "/fixture/persistent/test.plugin"} + manager.data = files + require.NoError(t, store.Save(t.Context(), managerInstallation(t, "test.plugin", true))) + require.NoError(t, manager.Sync(t.Context(), "test.plugin")) + require.Equal(t, files.path, processes.started[0].DataDir) + require.NotEqual(t, processes.started[0].StateDir, processes.started[0].DataDir) + require.NoError(t, service.Uninstall(t.Context(), "test.plugin", false)) + require.Empty(t, files.removed) +} + +// TestDataPurgeRetriesAfterCommittedUninstall injects a local deletion failure after host records commit. +// Retrying the explicit purge removes remaining files without resurrecting installation intent or changing the retained ledger policy. +func TestDataPurgeRetriesAfterCommittedUninstall(t *testing.T) { + manager, service, store, _, _ := managerFixture(t, 1) + files := &dataFixture{removeError: errors.New("fixture disk failure")} + manager.data = files + require.NoError(t, store.Save(t.Context(), managerInstallation(t, "test.plugin", false))) + require.ErrorIs(t, service.Uninstall(t.Context(), "test.plugin", true), files.removeError) + _, err := store.Get(t.Context(), "test.plugin") + require.ErrorIs(t, err, ErrPluginNotFound) + files.removeError = nil + require.NoError(t, service.PurgeData(t.Context(), "test.plugin")) + require.Equal(t, []string{"test.plugin", "test.plugin"}, files.removed) + require.Equal(t, []bool{true, true}, store.purge) +} + +// TestDataBackupRequiresDisabledIntentAndCompletedDrain varies desired state and callback completion. +// Enabled or incompletely drained owners never reach snapshot creation; a stopped owner can produce a snapshot afterward. +func TestDataBackupRequiresDisabledIntentAndCompletedDrain(t *testing.T) { + manager, service, store, processes, _ := managerFixture(t, 1) + files := &dataFixture{} + manager.data = files + item := managerInstallation(t, "test.plugin", true) + require.NoError(t, store.Save(t.Context(), item)) + _, err := service.BackupData(t.Context(), item.PluginID) + require.ErrorIs(t, err, ErrDataRequiresStop) + require.Zero(t, files.backups) + item.Enabled = false + require.NoError(t, store.Save(t.Context(), item)) + processes.stopError = errors.New("fixture callback still active") + _, err = service.BackupData(t.Context(), item.PluginID) + require.ErrorIs(t, err, processes.stopError) + require.Zero(t, files.backups) + processes.stopError = nil + archive, err := service.BackupData(t.Context(), item.PluginID) + require.NoError(t, err) + require.NoError(t, archive.Reader.Close()) + require.Equal(t, 1, files.backups) + require.ErrorIs(t, service.PurgeData(t.Context(), item.PluginID), ErrPluginAlreadyInstalled) + require.Empty(t, files.removed) +} diff --git a/backend/internal/plugin/doc.go b/backend/internal/plugin/doc.go index a6b31f8db..22f74fd16 100644 --- a/backend/internal/plugin/doc.go +++ b/backend/internal/plugin/doc.go @@ -11,6 +11,7 @@ // - catalog.go selects source-qualified releases; catalog_service.go, catalog_refresh.go, and catalog_indexes.go own cache and registration coordination. // - configuration.go, limits.go, and dev_source.go normalize persistent policy and development records. // - grant.go, ledger.go, wallet.go, redeem.go, subscriptions.go, and capabilities.go define caller authority, financial rules, and audit collaboration. +// - data.go coordinates persistent directories, retained-owner cleanup, and offline backup management. // - http.go defines transport-independent HTTP contribution values. // - tasks.go schedules declared jobs and retains in-memory execution results. // - kv.go and kv_maintenance.go enforce isolated JSON storage and sampled quota maintenance. diff --git a/backend/internal/plugin/errors.go b/backend/internal/plugin/errors.go index b88fae5a7..d4c98f90b 100644 --- a/backend/internal/plugin/errors.go +++ b/backend/internal/plugin/errors.go @@ -29,6 +29,11 @@ var ( ErrIndexNotFound = apperror.NotFound("PLUGIN_INDEX_NOT_FOUND", "Plugin index not found") ErrSecretKeyMissing = apperror.BadRequest("PLUGIN_SECRET_KEY_REQUIRED", "Configure a persistent security.secret_encryption_key before saving plugin secrets") ErrLimitsInvalid = apperror.BadRequest("PLUGIN_LIMITS_INVALID", "Plugin limits are invalid") + ErrDataRequiresStop = apperror.Conflict("PLUGIN_DATA_REQUIRES_STOP", "Disable the plugin before backing up or restoring its data") + ErrDataNotFound = apperror.NotFound("PLUGIN_DATA_NOT_FOUND", "Plugin data not found") + ErrDataExists = apperror.Conflict("PLUGIN_DATA_EXISTS", "Plugin data directory is not empty") + ErrDataInvalid = apperror.BadRequest("PLUGIN_DATA_INVALID", "Plugin data archive or directory is invalid") + ErrDataTooLarge = apperror.BadRequest("PLUGIN_DATA_TOO_LARGE", "Plugin data exceeds archive limits") ErrMaxRunning = apperror.Conflict("MAX_RUNNING_EXCEEDED", "Plugin process limit reached") ErrIndexSignatureInvalid = apperror.BadRequest("PLUGIN_INDEX_SIGNATURE_INVALID", "Plugin index signature is invalid") ErrConfigInvalid = apperror.BadRequest("PLUGIN_CONFIG_INVALID", "Plugin configuration is invalid") diff --git a/backend/internal/plugin/helpers_test.go b/backend/internal/plugin/helpers_test.go index bb9426d29..3ecb71934 100644 --- a/backend/internal/plugin/helpers_test.go +++ b/backend/internal/plugin/helpers_test.go @@ -278,7 +278,7 @@ func managerFixture(t *testing.T, maximum int) (*Manager, *Service, *memoryInsta store := newMemoryInstallations() processes := &processFixture{} packages := &packageFixture{} - manager := NewManager(ManagerOptions{Enabled: true, MaxRunning: maximum, HostVersion: "0.5.4", StateRoot: t.TempDir()}, store, packages, processes, NewConfiguration(nil, false), slog.New(slog.NewTextHandler(io.Discard, nil))) + manager := NewManager(ManagerOptions{Enabled: true, MaxRunning: maximum, HostVersion: "0.5.4", StateRoot: t.TempDir()}, store, packages, processes, NewConfiguration(nil, false), nil, slog.New(slog.NewTextHandler(io.Discard, nil))) return manager, NewService(manager, nil, nil, nil), store, processes, packages } diff --git a/backend/internal/plugin/httpapi/data.go b/backend/internal/plugin/httpapi/data.go new file mode 100644 index 000000000..f0af78cac --- /dev/null +++ b/backend/internal/plugin/httpapi/data.go @@ -0,0 +1,66 @@ +package httpapi + +import ( + "mime" + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/TokenFlux/TokenRouter/internal/plugin" + "github.com/TokenFlux/TokenRouter/internal/server/httpx" +) + +// data.go exposes administrator-only management of persistent plugin files. +// Inventory includes uninstalled owners, while backup and restore use the core +// service's stopped-owner checks. File archives remain separate from host SQL +// backups and ordinary plugin contribution request and response limits. + +// ListData returns retained directory usage without exposing absolute filesystem paths. +func (h *Handler) ListData(c *gin.Context) { + items, err := h.service.ListData(c.Request.Context()) + if httpx.ErrorFrom(c, err) { + return + } + httpx.Success(c, items) +} + +// PurgeData removes an uninstalled owner's local files and retained KV records. +func (h *Handler) PurgeData(c *gin.Context) { + if httpx.ErrorFrom(c, h.service.PurgeData(c.Request.Context(), c.Param("id"))) { + return + } + httpx.Success(c, nil) +} + +// BackupData streams a completed private snapshot after stopped-owner validation succeeds. +// The archive reader belongs to this response and is closed on interrupted +// delivery as well. Cache prevention applies to successful and failed responses. +func (h *Handler) BackupData(c *gin.Context) { + c.Header("Cache-Control", "no-store") + archive, err := h.service.BackupData(c.Request.Context(), c.Param("id")) + if httpx.ErrorFrom(c, err) { + return + } + defer func() { _ = archive.Reader.Close() }() + c.DataFromReader(http.StatusOK, archive.Bytes, "application/zip", archive.Reader, map[string]string{ + "Content-Disposition": mime.FormatMediaType("attachment", map[string]string{"filename": c.Param("id") + "-data.zip"}), + "X-Content-Type-Options": "nosniff", + }) +} + +// RestoreData accepts a bounded raw ZIP for an absent or empty stopped owner. +// Content validation and publication belong to the persistent file store. Failed +// archives return stable management errors and never overwrite existing business data. +func (h *Handler) RestoreData(c *gin.Context) { + media, _, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil || media != "application/zip" { + httpx.ErrorFrom(c, plugin.ErrDataInvalid) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, plugin.DataMaxArchiveBytes+1) + defer func() { _ = c.Request.Body.Close() }() + if httpx.ErrorFrom(c, h.service.RestoreData(c.Request.Context(), c.Param("id"), c.Request.Body)) { + return + } + httpx.Success(c, nil) +} diff --git a/backend/internal/plugin/httpapi/data_test.go b/backend/internal/plugin/httpapi/data_test.go new file mode 100644 index 000000000..f6119c767 --- /dev/null +++ b/backend/internal/plugin/httpapi/data_test.go @@ -0,0 +1,97 @@ +package httpapi + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/TokenFlux/TokenRouter/internal/plugin" + "github.com/TokenFlux/TokenRouter/internal/plugin/provider" +) + +// data_test.go exercises administrative file management through registered HTTP routes. +// Actual directory and archive adapters verify binary transfer, retained-owner +// discovery, empty-destination restore, and conflicts independently of contribution RPC limits. + +// dataInstallationStore represents an uninstalled owner for retained-data HTTP scenarios. +type dataInstallationStore struct { + plugin.InstallationStore // unused persistence operations +} + +// Get reports missing installation intent while local business files remain retained. +func (dataInstallationStore) Get(context.Context, string) (*plugin.Installation, error) { + return nil, plugin.ErrPluginNotFound +} + +// List reports an empty installation inventory independently of retained local directories. +func (dataInstallationStore) List(context.Context) ([]plugin.Installation, error) { + return nil, nil +} + +// Delete accepts repeated deletion of an already absent installation and its KV. +func (dataInstallationStore) Delete(context.Context, string, bool) error { return nil } + +// TestDataHTTPBackupAndRestoreTransfersLargePrivateFiles downloads a ZIP exceeding ordinary contribution limits. +// The response stays binary and noncacheable, and restoration reproduces its private bytes after an explicit retained-data purge. +func TestDataHTTPBackupAndRestoreTransfersLargePrivateFiles(t *testing.T) { + gin.SetMode(gin.TestMode) + base := t.TempDir() + files, err := provider.NewDataFiles(filepath.Join(base, "data")) + require.NoError(t, err) + directory, err := files.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + content := make([]byte, 5<<20) + _, err = rand.Read(content) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(directory, "private.bin"), content, 0o600)) + packages, err := provider.NewPackageFiles(base, provider.DownloadOptions{}) + require.NoError(t, err) + processes := provider.NewProcesses(plugin.RuntimeOptions{}, nil, nil) + manager := plugin.NewManager(plugin.ManagerOptions{}, dataInstallationStore{}, packages, processes, plugin.NewConfiguration(nil, false), files, nil) + service := plugin.NewService(manager, packages, nil, nil) + router := gin.New() + RegisterAdminRoutes(router.Group("/api/v1/admin"), NewHandler(service, manager, processes, nil), func(c *gin.Context) { c.Next() }) + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/admin/plugins/data", nil)) + require.Equal(t, http.StatusOK, response.Code) + var inventory struct { + Data []plugin.DataUsage `json:"data"` // returned retained data inventory + } + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &inventory)) + require.Len(t, inventory.Data, 1) + require.False(t, inventory.Data[0].Installed) + require.NotContains(t, response.Body.String(), base) + response = httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/admin/plugins/test.plugin/data/archive", nil)) + require.Equal(t, http.StatusOK, response.Code) + require.Equal(t, "application/zip", response.Header().Get("Content-Type")) + require.Equal(t, "no-store", response.Header().Get("Cache-Control")) + require.Greater(t, response.Body.Len(), 4<<20) + archive := bytes.Clone(response.Body.Bytes()) + request := httptest.NewRequest(http.MethodPost, "/api/v1/admin/plugins/test.plugin/data/archive", bytes.NewReader(archive)) + request.Header.Set("Content-Type", "application/zip") + response = httptest.NewRecorder() + router.ServeHTTP(response, request) + require.Equal(t, http.StatusConflict, response.Code) + require.Contains(t, response.Body.String(), "PLUGIN_DATA_EXISTS") + response = httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest(http.MethodDelete, "/api/v1/admin/plugins/test.plugin/data", nil)) + require.Equal(t, http.StatusOK, response.Code) + request = httptest.NewRequest(http.MethodPost, "/api/v1/admin/plugins/test.plugin/data/archive", bytes.NewReader(archive)) + request.Header.Set("Content-Type", "application/zip") + response = httptest.NewRecorder() + router.ServeHTTP(response, request) + require.Equal(t, http.StatusOK, response.Code) + restored, err := os.ReadFile(filepath.Join(directory, "private.bin")) + require.NoError(t, err) + require.Equal(t, content, restored) +} diff --git a/backend/internal/plugin/httpapi/management_operations.go b/backend/internal/plugin/httpapi/management_operations.go index 410b36ca1..1b14f5946 100644 --- a/backend/internal/plugin/httpapi/management_operations.go +++ b/backend/internal/plugin/httpapi/management_operations.go @@ -8,10 +8,61 @@ import ( // management_operations.go registers plugin administration in the shared MCP // operation catalog. Executable changes and capability policy edits declare // step-up requirements, which the route registrar also enforces for direct HTTP. -// Multipart upload uses a browser handoff because MCP requests carry JSON bodies. +// File uploads use a browser handoff because MCP requests carry JSON bodies. // ManagementOperations owns stable permission identities and schemas for every installed-plugin administrative route mounted by this module's registrar. var ManagementOperations = map[string]*adminapi.Operation{ + "plugins.data.list": adminapi.Define(adminapi.Operation{ + ID: "plugins.data.list", + Module: "plugins", + Title: "List retained plugin data", + Description: "List local plugin business directories, including uninstalled owners, with byte and file counts. Absolute paths and file contents are omitted.", + Method: "GET", + Path: "/api/v1/admin/plugins/data", + ReadOnly: true, + BrowserPath: "/admin/plugins?view=data", + Input: adminapi.Arguments("/api/v1/admin/plugins/data", nil, nil), + Output: adminapi.JSONResponse((*[]plugin.DataUsage)(nil)), + }), + "plugins.data.delete": adminapi.Define(adminapi.Operation{ + ID: "plugins.data.delete", + Module: "plugins", + Title: "Purge retained plugin data", + Description: "Delete an uninstalled plugin's local business files and KV. A concurrent reinstall is excluded; financial and delivery ledgers remain retained.", + Method: "DELETE", + Path: "/api/v1/admin/plugins/:id/data", + Sensitive: true, + StepUp: true, + BrowserPath: "/admin/plugins?view=data", + Input: adminapi.Arguments("/api/v1/admin/plugins/:id/data", nil, nil), + }), + "plugins.data.backup": adminapi.Define(adminapi.Operation{ + ID: "plugins.data.backup", + Module: "plugins", + Title: "Download plugin data backup", + Description: "Download a checksummed ZIP of local files for a disabled or uninstalled plugin. Host SQL, configuration, and external storage require separate backups.", + Method: "GET", + Path: "/api/v1/admin/plugins/:id/data/archive", + ReadOnly: true, + Sensitive: true, + StepUp: true, + Response: adminapi.ResponseFile, + BrowserPath: "/admin/plugins?view=data", + Input: adminapi.Arguments("/api/v1/admin/plugins/:id/data/archive", nil, nil), + }), + "plugins.data.restore": adminapi.Define(adminapi.Operation{ + ID: "plugins.data.restore", + Module: "plugins", + Title: "Restore plugin data backup", + Description: "Upload a raw application/zip backup for its matching plugin ID while disabled or uninstalled. The destination must be absent or empty; nonempty data is never overwritten.", + Method: "POST", + Path: "/api/v1/admin/plugins/:id/data/archive", + Sensitive: true, + StepUp: true, + Response: adminapi.ResponseBrowser, + BrowserPath: "/admin/plugins?view=data", + Input: adminapi.Arguments("/api/v1/admin/plugins/:id/data/archive", nil, nil), + }), "plugins.list": adminapi.Define(adminapi.Operation{ ID: "plugins.list", Module: "plugins", @@ -103,7 +154,7 @@ var ManagementOperations = map[string]*adminapi.Operation{ ID: "plugins.delete", Module: "plugins", Title: "Uninstall plugin", - Description: "Drain executable calls and host callbacks, then remove the installation. purge_data=true also deletes its KV data in the same transaction. Financial and delivery ledger history remains.", + Description: "Drain executable calls and host callbacks, then remove the installation. purge_data=true also deletes its KV in the same transaction and then removes local business files; failed file cleanup remains retryable. Financial and delivery ledger history remains.", Method: "DELETE", Path: "/api/v1/admin/plugins/:id", Sensitive: true, diff --git a/backend/internal/plugin/httpapi/routes_admin.go b/backend/internal/plugin/httpapi/routes_admin.go index 214ddd315..e755d15dd 100644 --- a/backend/internal/plugin/httpapi/routes_admin.go +++ b/backend/internal/plugin/httpapi/routes_admin.go @@ -17,6 +17,8 @@ import ( // @project-doc docs/interfaces/plugin_contracts.md#administration func RegisterAdminRoutes(admin *gin.RouterGroup, handler *Handler, stepUp gin.HandlerFunc) { for id, endpoint := range map[string]gin.HandlerFunc{ + "plugins.data.list": handler.ListData, "plugins.data.delete": handler.PurgeData, + "plugins.data.backup": handler.BackupData, "plugins.data.restore": handler.RestoreData, "plugins.update": handler.Update, "plugins.rollback": handler.Rollback, "plugins.catalog.list": handler.Catalog, "plugins.catalog.get": handler.CatalogEntry, "plugins.catalog.refresh": handler.RefreshCatalog, "plugins.indexes.list": handler.Indexes, "plugins.indexes.create": handler.CreateIndex, "plugins.indexes.update": handler.UpdateIndex, "plugins.indexes.delete": handler.DeleteIndex, diff --git a/backend/internal/plugin/manager.go b/backend/internal/plugin/manager.go index e94ed3799..6d9331f74 100644 --- a/backend/internal/plugin/manager.go +++ b/backend/internal/plugin/manager.go @@ -48,6 +48,7 @@ type Loaded struct { type Manager struct { options ManagerOptions // host plugin execution policy store InstallationStore // durable and development installation source + data DataStorage // durable plugin-owned files and offline archives packages Packages // verified local archive storage processes Supervisors // child generation lifecycle owner configuration *Configuration // schema and encrypted settings boundary @@ -71,7 +72,15 @@ type Manager struct { // NewManager constructs a dormant reconciler with no filesystem or process effects. // Installation sources and configuration are shared dependencies. Start loads records // asynchronously; callers can inspect empty snapshots before initialization completes. -func NewManager(options ManagerOptions, store InstallationStore, packages Packages, processes Supervisors, configuration *Configuration, logger *slog.Logger) *Manager { +func NewManager( + options ManagerOptions, + store InstallationStore, + packages Packages, + processes Supervisors, + configuration *Configuration, + data DataStorage, + logger *slog.Logger, +) *Manager { if logger == nil { logger = slog.Default() } @@ -79,6 +88,7 @@ func NewManager(options ManagerOptions, store InstallationStore, packages Packag options: options, store: store, packages: packages, + data: data, processes: processes, configuration: configuration, logger: logger, @@ -303,12 +313,22 @@ func (m *Manager) syncLocked(ctx context.Context, id string, retain ...PackageRe if err != nil { return err } + dataDir := "" + if m.data != nil { + // Prepare the persistent owner before starting executable code, keeping + // business files independent of versioned packages and disposable process state. + dataDir, err = m.data.Directory(ctx, id) + if err != nil { + return err + } + } spec := ProcessSpec{ PluginID: id, Version: want.Version, Executable: local.Executable, BinarySHA256: local.BinarySHA256, StateDir: stateDir, + DataDir: dataDir, ConfigJSON: data, Permissions: slices.Clone(want.GrantedPermissions), MemoryMiB: want.Manifest.Resources.MemoryLimit(), diff --git a/backend/internal/plugin/postgres/source_removal_scenario_integration_test.go b/backend/internal/plugin/postgres/source_removal_scenario_integration_test.go index 23af79292..307472045 100644 --- a/backend/internal/plugin/postgres/source_removal_scenario_integration_test.go +++ b/backend/internal/plugin/postgres/source_removal_scenario_integration_test.go @@ -99,7 +99,7 @@ func TestUpdateRestoresHistoryAfterSourceRemoval(t *testing.T) { } manager := plugin.NewManager(plugin.ManagerOptions{ Enabled: true, MaxRunning: 1, HostVersion: "0.5.4", StateRoot: t.TempDir(), - }, store, sourceRemovalPackages{}, processes, plugin.NewConfiguration(nil, false), logger) + }, store, sourceRemovalPackages{}, processes, plugin.NewConfiguration(nil, false), nil, logger) require.NoError(t, manager.Sync(t.Context(), original.PluginID)) service := plugin.NewService(manager, nil, sourceRemovalResolver{target: candidate}, nil) // Candidate replacement deletes its source after persistence, forcing recovery diff --git a/backend/internal/plugin/provider/data_archives.go b/backend/internal/plugin/provider/data_archives.go new file mode 100644 index 000000000..1e8e6bc94 --- /dev/null +++ b/backend/internal/plugin/provider/data_archives.go @@ -0,0 +1,454 @@ +package provider + +import ( + "archive/zip" + "compress/flate" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "github.com/TokenFlux/TokenRouter/internal/plugin" +) + +// data_archives.go snapshots stopped plugin files and validates portable restores. +// Each ZIP binds its plugin ID and relative file names to a versioned manifest +// with byte counts and SHA-256 digests. Private temporary files keep complete +// snapshots independent of later plugin execution and response interruptions. +// +// Restore first receives and validates the complete archive in bounded staging. +// Files are extracted beneath a scoped root, with links, duplicate names, special +// files, unexpected entries, and excessive expansion rejected. Publication renames +// that completed directory into an absent or empty destination, leaving existing +// business files untouched on validation failure. The manager keeps execution +// stopped and excludes reinstall, purge, and enable throughout this operation. + +const ( + // Archive limits bound disk consumption and metadata processing for administrator + // imports and exports, independently of ordinary plugin contribution HTTP limits. + dataMaxBytes = plugin.DataMaxArchiveBytes + dataMaxEntries = 10000 + dataMaxManifestBytes = 4 << 20 +) + +// dataManifest binds the exported business files to one plugin identity. +type dataManifest struct { + Format int `json:"format"` // archive format version + PluginID string `json:"plugin_id"` // persistent data owner + Entries []dataEntry `json:"entries"` // ordered files and directories +} + +// dataEntry records one portable path and its expected restored content. +type dataEntry struct { + Path string `json:"path"` // slash-separated relative path + // Directory, Bytes, and SHA256 describe the same entry. Directories have zero + // bytes and no digest; regular files declare their decoded size and content hash. + Directory bool `json:"directory,omitempty"` // entry represents directory + Bytes int64 `json:"bytes"` // uncompressed regular file bytes + SHA256 string `json:"sha256,omitempty"` // hexadecimal file content digest +} + +// dataArchiveFile removes a temporary snapshot when its response reader closes. +type dataArchiveFile struct { + *os.File // owned immutable snapshot file +} + +// dataContextReader observes cancellation between bounded archive read operations. +type dataContextReader struct { + ctx context.Context // archive operation cancellation + reader io.Reader // underlying bounded input +} + +// Backup builds a complete checksummed snapshot before exposing its response stream. +// Callers keep the plugin stopped during creation. Snapshot files are private, +// and closing the returned reader releases them after delivery or cancellation. +// +// Files and their content identities are collected under one stopped-owner lock. +// The ZIP central directory is finalized before the result becomes visible, so +// disk failures cannot produce a successful partial download. A later enable can +// resume execution after snapshot creation because subsequent response reads use +// the completed temporary file rather than live plugin business files. +func (d *DataFiles) Backup(ctx context.Context, id string) (*plugin.DataArchive, error) { + owner, err := d.ownerRoot(ctx, id, false) + if err != nil { + return nil, err + } + defer func() { _ = owner.Close() }() + file, err := os.CreateTemp(d.root, ".data-snapshot-*.zip") + if err != nil { + return nil, err + } + archive := &dataArchiveFile{File: file} + complete := false + // Until the ZIP is finalized, this call owns temporary-file cleanup; failed + // cleanup remains under the reserved startup-reclaimed prefix for later removal. + defer func() { + if !complete { + _ = archive.Close() + } + }() + writer := zip.NewWriter(file) + metadata := dataManifest{Format: 1, PluginID: id, Entries: []dataEntry{}} + var total int64 + err = fs.WalkDir(owner.FS(), ".", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + if name == "." { + return nil + } + if !validDataPath(name) { + return plugin.ErrDataInvalid + } + if len(metadata.Entries) >= dataMaxEntries { + return plugin.ErrDataTooLarge + } + info, err := entry.Info() + if err != nil { + return err + } + item := dataEntry{Path: name, Directory: info.IsDir()} + archiveName := "data/" + name + if item.Directory { + archiveName += "/" + } else if !info.Mode().IsRegular() { + return plugin.ErrDataInvalid + } + if info.Size() < 0 || (!item.Directory && info.Size() > dataMaxBytes-total) { + return plugin.ErrDataTooLarge + } + destination, err := writer.Create(archiveName) + if err != nil { + return err + } + if !item.Directory { + item.Bytes, item.SHA256, err = writeDataFile(ctx, owner, name, destination, info.Size()) + if err != nil { + return err + } + total += item.Bytes + } + metadata.Entries = append(metadata.Entries, item) + return nil + }) + if err != nil { + return nil, err + } + encoded, err := json.Marshal(metadata) + if err != nil { + return nil, err + } + if len(encoded) > dataMaxManifestBytes { + return nil, plugin.ErrDataTooLarge + } + destination, err := writer.Create("manifest.json") + if err != nil { + return nil, err + } + if _, err := destination.Write(encoded); err != nil { + return nil, err + } + // Finish the central directory before returning a snapshot, so write errors + // cannot turn a partially transferred archive into a successful management response. + if err := writer.Close(); err != nil { + return nil, err + } + info, err := file.Stat() + if err != nil { + return nil, err + } + if info.Size() > dataMaxBytes { + return nil, plugin.ErrDataTooLarge + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + complete = true + return &plugin.DataArchive{Reader: archive, Bytes: info.Size()}, nil +} + +// writeDataFile copies a bounded stopped file and returns its exact content identity. +// The size observed before opening must match the copied bytes. The caller owns +// the archive writer, while this function closes its scoped source descriptor. +func writeDataFile(ctx context.Context, root *os.Root, name string, destination io.Writer, size int64) (int64, string, error) { + source, err := root.Open(name) + if err != nil { + return 0, "", err + } + defer func() { _ = source.Close() }() + digest := sha256.New() + written, err := io.Copy(io.MultiWriter(destination, digest), io.LimitReader(dataContextReader{ctx: ctx, reader: source}, size+1)) + if err != nil { + return 0, "", err + } + if written != size { + return 0, "", plugin.ErrDataInvalid + } + return written, hex.EncodeToString(digest.Sum(nil)), nil +} + +// Restore validates an archive and publishes its files into an absent or empty owner. +// The complete upload is spooled before ZIP inspection, and extraction occurs in +// a private directory on the destination filesystem. Existing nonempty data always +// returns a conflict; administrators retain its backup before removing that owner. +// +// The final rename publishes only files whose manifest identity, byte counts, +// checksums, paths, and expansion limits have passed. Validation or disk failures +// discard staging and leave destination contents untouched. Callers serialize +// this operation with plugin execution and installation changes for the same ID. +func (d *DataFiles) Restore(ctx context.Context, id string, source io.Reader) error { + if _, err := d.destination(ctx, id); err != nil { + return err + } + file, err := os.CreateTemp(d.root, ".data-upload-*.zip") + if err != nil { + return err + } + defer func() { _ = (&dataArchiveFile{File: file}).Close() }() + written, err := io.Copy(file, io.LimitReader(dataContextReader{ctx: ctx, reader: source}, dataMaxBytes+1)) + if err != nil { + return err + } + if written > dataMaxBytes { + return plugin.ErrDataTooLarge + } + archive, err := zip.NewReader(file, written) + if err != nil { + return plugin.ErrDataInvalid.WithCause(err) + } + metadata, entries, err := readDataManifest(archive, id) + if err != nil { + return err + } + stage, err := os.MkdirTemp(d.root, ".data-restore-") + if err != nil { + return err + } + // Failed extraction keeps only private staging. Cleanup failures leave its + // reserved name for startup reclamation while preserving the original restore error. + defer func() { _ = os.RemoveAll(stage) }() + root, err := os.OpenRoot(stage) + if err != nil { + return err + } + err = extractData(ctx, root, metadata, entries) + closeErr := root.Close() + if err := errors.Join(err, closeErr); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + empty, err := d.destination(ctx, id) + if err != nil { + return err + } + parent, err := os.OpenRoot(d.root) + if err != nil { + return err + } + defer func() { _ = parent.Close() }() + if empty { + // Removing an empty directory supports Windows rename behavior; this can + // lose only an empty placeholder if publication fails, never existing business files. + if err := parent.Remove(id); err != nil { + return err + } + } + return parent.Rename(filepath.Base(stage), id) +} + +// destination rejects populated or linked owner directories before restore publication. +// Its boolean reports an existing empty directory. Missing destinations are valid, +// and the caller rechecks under the management lock immediately before rename. +func (d *DataFiles) destination(ctx context.Context, id string) (bool, error) { + owner, err := d.ownerRoot(ctx, id, false) + if errors.Is(err, plugin.ErrDataNotFound) { + return false, nil + } + if err != nil { + return false, err + } + defer func() { _ = owner.Close() }() + entries, err := fs.ReadDir(owner.FS(), ".") + if err != nil { + return false, err + } + if len(entries) > 0 { + return false, plugin.ErrDataExists + } + return true, nil +} + +// readDataManifest checks owner identity and the archive's complete declared entry set. +// Only a bounded version-one manifest is accepted. Duplicate names, file aliases, +// links, unsupported modes, and undeclared ZIP entries fail before extraction begins. +// +// The manifest must account for every ZIP member exactly once so aliases and +// additional payloads cannot bypass content checks. Expanded byte counts are +// accumulated before file creation, and each regular file must declare a SHA-256 +// digest that extraction later compares against its actual decompressed bytes. +func readDataManifest(archive *zip.Reader, id string) (dataManifest, map[string]*zip.File, error) { + var metadata dataManifest + if len(archive.File) > dataMaxEntries+1 { + return metadata, nil, plugin.ErrDataTooLarge + } + if len(archive.File) == 0 { + return metadata, nil, plugin.ErrDataInvalid + } + entries := make(map[string]*zip.File, len(archive.File)) + for _, entry := range archive.File { + if _, duplicate := entries[entry.Name]; duplicate || entry.Mode()&^(os.ModePerm|os.ModeDir) != 0 { + return metadata, nil, plugin.ErrDataInvalid + } + entries[entry.Name] = entry + } + manifestFile, ok := entries["manifest.json"] + if !ok || manifestFile.UncompressedSize64 > dataMaxManifestBytes { + return metadata, nil, plugin.ErrDataInvalid + } + reader, err := manifestFile.Open() + if err != nil { + return metadata, nil, plugin.ErrDataInvalid.WithCause(err) + } + encoded, err := io.ReadAll(io.LimitReader(reader, dataMaxManifestBytes+1)) + closeErr := reader.Close() + if err := errors.Join(err, closeErr); err != nil { + return metadata, nil, plugin.ErrDataInvalid.WithCause(err) + } + if len(encoded) > dataMaxManifestBytes || + json.Unmarshal(encoded, &metadata) != nil || + metadata.Format != 1 || metadata.PluginID != id || + len(metadata.Entries) != len(entries)-1 { + return metadata, nil, plugin.ErrDataInvalid + } + seen := make(map[string]bool, len(metadata.Entries)) + var total int64 + for _, item := range metadata.Entries { + if !validDataPath(item.Path) || seen[item.Path] || item.Bytes < 0 || item.Bytes > dataMaxBytes-total { + return metadata, nil, plugin.ErrDataInvalid + } + seen[item.Path] = true + name := "data/" + item.Path + if item.Directory { + name += "/" + if item.Bytes != 0 || item.SHA256 != "" { + return metadata, nil, plugin.ErrDataInvalid + } + } else { + digest, err := hex.DecodeString(item.SHA256) + if err != nil || len(digest) != sha256.Size { + return metadata, nil, plugin.ErrDataInvalid + } + } + entry, ok := entries[name] + if !ok || entry.UncompressedSize64 != uint64(item.Bytes) || + entry.FileInfo().IsDir() != item.Directory || + (!item.Directory && !entry.Mode().IsRegular()) { + return metadata, nil, plugin.ErrDataInvalid + } + total += item.Bytes + } + return metadata, entries, nil +} + +// extractData restores verified entries beneath a private root and checks content hashes. +// Callers own publication and discard the entire staging directory on error. File +// bytes are bounded again during decompression, independently of claimed ZIP sizes. +func extractData(ctx context.Context, root *os.Root, metadata dataManifest, entries map[string]*zip.File) error { + for _, item := range metadata.Entries { + if err := ctx.Err(); err != nil { + return err + } + if item.Directory { + if err := root.MkdirAll(item.Path, 0o700); err != nil { + return err + } + continue + } + if err := root.MkdirAll(path.Dir(item.Path), 0o700); err != nil { + return err + } + if err := extractDataFile(ctx, root, item, entries["data/"+item.Path]); err != nil { + return err + } + } + return nil +} + +// extractDataFile writes one bounded regular file and validates its expected SHA-256. +// Descriptors are closed before returning so publication never includes open staged +// writers. Disk and checksum failures leave cleanup to the enclosing restore operation. +// +// Archive metadata alone cannot establish the decoded content: the copy bounds +// actual bytes and computes their digest while writing. Before publication, the +// byte count and digest are compared with the manifest before flushing and +// closing the destination, so failed writeback or corrupt compressed input cannot expose +// a seemingly successful restored business file. +func extractDataFile(ctx context.Context, root *os.Root, item dataEntry, entry *zip.File) error { + source, err := entry.Open() + if err != nil { + return plugin.ErrDataInvalid.WithCause(err) + } + defer func() { _ = source.Close() }() + destination, err := root.OpenFile(item.Path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + digest := sha256.New() + written, copyErr := io.Copy(io.MultiWriter(destination, digest), io.LimitReader(dataContextReader{ctx: ctx, reader: source}, item.Bytes+1)) + if copyErr == nil && (written != item.Bytes || !strings.EqualFold(hex.EncodeToString(digest.Sum(nil)), item.SHA256)) { + copyErr = plugin.ErrDataInvalid + } + var malformed flate.CorruptInputError + if errors.As(copyErr, &malformed) || errors.Is(copyErr, zip.ErrChecksum) || + errors.Is(copyErr, zip.ErrFormat) || errors.Is(copyErr, io.ErrUnexpectedEOF) { + // Damaged compressed input is a bad archive; preserve write and cancellation + // errors separately so storage failures are not reported as invalid uploads. + copyErr = plugin.ErrDataInvalid.WithCause(copyErr) + } + var syncErr error + if copyErr == nil { + // Flush validated bytes before publication so writeback errors cannot follow + // a successful restore, while corrupt partial files need no additional writeback. + syncErr = destination.Sync() + } + closeErr := destination.Close() + return errors.Join(copyErr, syncErr, closeErr) +} + +// validDataPath accepts bounded relative paths for archive business files. +// Path metadata and directory depth are capped across platforms. Separators and +// colon syntax that could change ownership during extraction are rejected. +func validDataPath(name string) bool { + return name != "." && len(name) <= 512 && strings.Count(name, "/") < 20 && + fs.ValidPath(name) && path.Clean(name) == name && + !strings.ContainsAny(name, "\\:\x00") +} + +// Close releases the snapshot descriptor and removes its private temporary file. +// Callers invoke it once after delivery or cancellation. Removal is attempted +// even if descriptor closing fails, and both errors remain available to callers. +func (f *dataArchiveFile) Close() error { + name := f.Name() + closeErr := f.File.Close() + return errors.Join(closeErr, os.Remove(name)) +} + +// Read observes archive cancellation before delegating each bounded input read. +func (r dataContextReader) Read(buffer []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} diff --git a/backend/internal/plugin/provider/data_archives_test.go b/backend/internal/plugin/provider/data_archives_test.go new file mode 100644 index 000000000..13eb85451 --- /dev/null +++ b/backend/internal/plugin/provider/data_archives_test.go @@ -0,0 +1,220 @@ +package provider + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "testing/iotest" + + "github.com/stretchr/testify/require" + + "github.com/TokenFlux/TokenRouter/internal/plugin" +) + +// data_archives_test.go exercises checksummed snapshots using actual ZIP and files. +// Round trips preserve binary payloads, SQLite companion files, and empty directories. +// Corrupt or hostile uploads must leave the destination unpublished or unchanged. + +// TestDataArchivesRoundTripStoppedFiles snapshots binary files and an empty directory. +// Restoring into a fresh store reproduces all bytes, and an already populated destination rejects replacement. +func TestDataArchivesRoundTripStoppedFiles(t *testing.T) { + source, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + directory, err := source.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + files := map[string][]byte{ + "plugin.sqlite": {0, 1, 2, 0xff}, + "plugin.sqlite-wal": []byte("committed WAL records"), + "images/private.png": []byte("private attachment bytes"), + } + for name, content := range files { + filename := filepath.Join(directory, filepath.FromSlash(name)) + require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0o700)) + require.NoError(t, os.WriteFile(filename, content, 0o600)) + } + require.NoError(t, os.Mkdir(filepath.Join(directory, "empty"), 0o700)) + archive, err := source.Backup(t.Context(), "test.plugin") + require.NoError(t, err) + content, err := io.ReadAll(archive.Reader) + require.NoError(t, err) + require.Equal(t, archive.Bytes, int64(len(content))) + require.NoError(t, archive.Reader.Close()) + target, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + require.NoError(t, target.Restore(t.Context(), "test.plugin", bytes.NewReader(content))) + for name, expected := range files { + actual, err := os.ReadFile(filepath.Join(target.root, "test.plugin", filepath.FromSlash(name))) + require.NoError(t, err) + require.Equal(t, expected, actual) + } + require.DirExists(t, filepath.Join(target.root, "test.plugin", "empty")) + require.ErrorIs(t, target.Restore(t.Context(), "test.plugin", bytes.NewReader(content)), plugin.ErrDataExists) + items, err := os.ReadDir(source.root) + require.NoError(t, err) + require.Len(t, items, 1) +} + +// TestDataArchivesRejectInvalidContent builds archives with traversal, identity, and digest violations. +// Every validation failure keeps the owner's existing empty directory free of partially restored files. +func TestDataArchivesRejectInvalidContent(t *testing.T) { + content := []byte("business data") + digest := sha256.Sum256(content) + valid := dataEntry{Path: "plugin.sqlite", Bytes: int64(len(content)), SHA256: hex.EncodeToString(digest[:])} + for _, test := range []struct { + name string // failing archive scenario + owner string // declared archive owner + entry dataEntry // untrusted manifest entry + extra bool // append undeclared payload + }{ + {name: "wrong owner", owner: "test.other", entry: valid}, + {name: "traversal", owner: "test.plugin", entry: dataEntry{Path: "../escape", Bytes: valid.Bytes, SHA256: valid.SHA256}}, + {name: "absolute", owner: "test.plugin", entry: dataEntry{Path: "/escape", Bytes: valid.Bytes, SHA256: valid.SHA256}}, + {name: "backslash", owner: "test.plugin", entry: dataEntry{Path: "dir\\escape", Bytes: valid.Bytes, SHA256: valid.SHA256}}, + {name: "checksum", owner: "test.plugin", entry: dataEntry{Path: valid.Path, Bytes: valid.Bytes, SHA256: hex.EncodeToString(make([]byte, sha256.Size))}}, + {name: "size", owner: "test.plugin", entry: dataEntry{Path: valid.Path, Bytes: valid.Bytes + 1, SHA256: valid.SHA256}}, + {name: "expansion", owner: "test.plugin", entry: dataEntry{Path: valid.Path, Bytes: dataMaxBytes + 1, SHA256: valid.SHA256}}, + {name: "extra file", owner: "test.plugin", entry: valid, extra: true}, + } { + t.Run(test.name, func(t *testing.T) { + target, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + directory, err := target.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + archive := dataTestArchive(t, dataManifest{Format: 1, PluginID: test.owner, Entries: []dataEntry{test.entry}}, map[string][]byte{"data/" + test.entry.Path: content}, test.extra) + require.Error(t, target.Restore(t.Context(), "test.plugin", bytes.NewReader(archive))) + entries, err := os.ReadDir(directory) + require.NoError(t, err) + require.Empty(t, entries) + entries, err = os.ReadDir(target.root) + require.NoError(t, err) + require.Len(t, entries, 1) + }) + } +} + +// TestDataArchivesRejectLinkedFiles creates a link to a different owner's private file. +// Backup rejects the link and removes its temporary snapshot, leaving the target's original bytes untouched. +func TestDataArchivesRejectLinkedFiles(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symbolic links requires platform-specific privileges") + } + data, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + directory, err := data.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + other, err := data.Directory(t.Context(), "test.other") + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(other, "private"), []byte("secret"), 0o600)) + require.NoError(t, os.Symlink(filepath.Join(other, "private"), filepath.Join(directory, "alias"))) + _, err = data.Backup(t.Context(), "test.plugin") + require.ErrorIs(t, err, plugin.ErrDataInvalid) + entries, err := os.ReadDir(data.root) + require.NoError(t, err) + require.Len(t, entries, 2) +} + +// TestDataArchivesRejectDuplicateAndLinkedMembers supplies ambiguous or executable filesystem metadata. +// Restore rejects duplicate member names, symbolic links, and device modes before publishing any destination files. +func TestDataArchivesRejectDuplicateAndLinkedMembers(t *testing.T) { + content := []byte("payload") + digest := sha256.Sum256(content) + metadata := dataManifest{Format: 1, PluginID: "test.plugin", Entries: []dataEntry{{Path: "file", Bytes: int64(len(content)), SHA256: hex.EncodeToString(digest[:])}}} + for _, mode := range []os.FileMode{0, os.ModeSymlink, os.ModeDevice} { + t.Run(mode.String(), func(t *testing.T) { + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + header := &zip.FileHeader{Name: "data/file"} + header.SetMode(mode | 0o600) + entry, err := writer.CreateHeader(header) + require.NoError(t, err) + _, err = entry.Write(content) + require.NoError(t, err) + if mode == 0 { + _, err := writer.Create("data/file") + require.NoError(t, err) + } + entry, err = writer.Create("manifest.json") + require.NoError(t, err) + require.NoError(t, json.NewEncoder(entry).Encode(metadata)) + require.NoError(t, writer.Close()) + target, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + require.ErrorIs(t, target.Restore(t.Context(), "test.plugin", bytes.NewReader(buffer.Bytes())), plugin.ErrDataInvalid) + require.NoDirExists(t, filepath.Join(target.root, "test.plugin")) + }) + } +} + +// TestDataArchivesInterruptedUploadLeavesExistingDirectoryEmpty fails after receiving a partial upload. +// The original read error propagates and private upload staging is removed, allowing a later complete restore to retry safely. +func TestDataArchivesInterruptedUploadLeavesExistingDirectoryEmpty(t *testing.T) { + target, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + directory, err := target.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + reader := io.MultiReader(bytes.NewReader([]byte("partial ZIP")), iotest.ErrReader(io.ErrUnexpectedEOF)) + require.ErrorIs(t, target.Restore(t.Context(), "test.plugin", reader), io.ErrUnexpectedEOF) + entries, err := os.ReadDir(directory) + require.NoError(t, err) + require.Empty(t, entries) + entries, err = os.ReadDir(target.root) + require.NoError(t, err) + require.Len(t, entries, 1) +} + +// TestDataArchivesCorruptZIPChecksumReturnsValidationError changes stored payload bytes after archive creation. +// The decoded member fails ZIP integrity checks, returns the stable invalid-archive error, and never publishes a business directory. +func TestDataArchivesCorruptZIPChecksumReturnsValidationError(t *testing.T) { + content := []byte("unique stopped SQLite payload") + digest := sha256.Sum256(content) + metadata := dataManifest{Format: 1, PluginID: "test.plugin", Entries: []dataEntry{{Path: "file", Bytes: int64(len(content)), SHA256: hex.EncodeToString(digest[:])}}} + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + entry, err := writer.CreateHeader(&zip.FileHeader{Name: "data/file", Method: zip.Store}) + require.NoError(t, err) + _, err = entry.Write(content) + require.NoError(t, err) + entry, err = writer.Create("manifest.json") + require.NoError(t, err) + require.NoError(t, json.NewEncoder(entry).Encode(metadata)) + require.NoError(t, writer.Close()) + archive := buffer.Bytes() + position := bytes.Index(archive, content) + require.GreaterOrEqual(t, position, 0) + archive[position] ^= 0xff + target, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + require.ErrorIs(t, target.Restore(t.Context(), "test.plugin", bytes.NewReader(archive)), plugin.ErrDataInvalid) + require.NoDirExists(t, filepath.Join(target.root, "test.plugin")) +} + +// dataTestArchive builds a synthetic archive with supplied metadata and untrusted names. +// Inputs intentionally bypass production export validation so restore checks encounter +// corrupted content and undeclared entries from an external archive producer. +func dataTestArchive(t *testing.T, metadata dataManifest, files map[string][]byte, extra bool) []byte { + t.Helper() + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + for name, content := range files { + entry, err := writer.Create(name) + require.NoError(t, err) + _, err = entry.Write(content) + require.NoError(t, err) + } + if extra { + _, err := writer.Create("data/undeclared") + require.NoError(t, err) + } + entry, err := writer.Create("manifest.json") + require.NoError(t, err) + require.NoError(t, json.NewEncoder(entry).Encode(metadata)) + require.NoError(t, writer.Close()) + return buffer.Bytes() +} diff --git a/backend/internal/plugin/provider/data_files.go b/backend/internal/plugin/provider/data_files.go new file mode 100644 index 000000000..8f9997403 --- /dev/null +++ b/backend/internal/plugin/provider/data_files.go @@ -0,0 +1,181 @@ +package provider + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/TokenFlux/TokenRouter/internal/plugin" + "github.com/TokenFlux/TokenRouter/plugin-sdk/go/manifest" +) + +// data_files.go creates and measures persistent directories owned by plugin IDs. +// These directories live separately from installed packages and disposable state, +// so package cleanup leaves business files available to a later installation. +// The manager serializes creation, restore, and purge for each owner. +// +// Filesystem operations use scoped root handles. Owner directories cannot be +// symbolic links, and usage measurements reject links and special files instead +// of traversing another owner's storage. This limits host management operations; +// administrator-trusted plugins still run with the host operating-system user. +// Temporary snapshot and restore names are reserved outside valid plugin IDs and +// reclaimed during startup before management requests can create active archives. + +// DataFiles owns persistent local directories independently of package installation. +type DataFiles struct { + root string // absolute persistent storage root +} + +// NewDataFiles prepares durable storage and removes interrupted archive staging. +// Construct one instance before admitting plugin administration. The root belongs +// to the host, while valid plugin-ID children survive initialization and uninstall. +func NewDataFiles(directory string) (*DataFiles, error) { + absolute, err := filepath.Abs(directory) + if err != nil { + return nil, err + } + if err := os.MkdirAll(absolute, 0o700); err != nil { + return nil, err + } + root, err := os.OpenRoot(absolute) + if err != nil { + return nil, err + } + defer func() { _ = root.Close() }() + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + return nil, err + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".data-") { + // Only host-reserved names are discarded; valid owner directories keep + // business files even when no installation record exists at this startup. + if err := root.RemoveAll(entry.Name()); err != nil { + return nil, err + } + } + } + return &DataFiles{root: absolute}, nil +} + +// Directory creates a private persistent directory for one validated plugin ID. +// Callers hold the owner's management lock through process initialization. Existing +// links and nondirectories are rejected before an executable receives the path. +func (d *DataFiles) Directory(ctx context.Context, id string) (string, error) { + root, err := d.ownerRoot(ctx, id, true) + if err != nil { + return "", err + } + if err := root.Close(); err != nil { + return "", err + } + return filepath.Join(d.root, id), nil +} + +// List measures retained owners while keeping temporary archive staging private. +// Directory measurements may change during running plugin writes. A malformed +// owner fails the request rather than reporting a misleading complete total. +func (d *DataFiles) List(ctx context.Context) ([]plugin.DataUsage, error) { + root, err := os.OpenRoot(d.root) + if err != nil { + return nil, err + } + defer func() { _ = root.Close() }() + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + return nil, err + } + items := make([]plugin.DataUsage, 0, len(entries)) + for _, entry := range entries { + if !manifest.ValidID(entry.Name()) { + continue + } + owner, err := d.ownerRoot(ctx, entry.Name(), false) + if err != nil { + return nil, err + } + usage := plugin.DataUsage{PluginID: entry.Name()} + err = fs.WalkDir(owner.FS(), ".", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return plugin.ErrDataInvalid + } + usage.Bytes += info.Size() + usage.Files++ + return nil + }) + closeErr := owner.Close() + if err := errors.Join(err, closeErr); err != nil { + return nil, err + } + items = append(items, usage) + } + return items, nil +} + +// Remove deletes only the validated owner's directory and succeeds when absent. +// Callers drain executable and host-callback work first. A symlink at the owner +// name is removed as a link; its target and other owners remain untouched. +func (d *DataFiles) Remove(ctx context.Context, id string) error { + if !manifest.ValidID(id) { + return plugin.ErrInvalidArgument + } + if err := ctx.Err(); err != nil { + return err + } + root, err := os.OpenRoot(d.root) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + return root.RemoveAll(id) +} + +// ownerRoot opens a validated owner directory, optionally creating an absent one. +// The returned handle belongs to the caller. Links and nondirectories fail, and +// missing data is reported with the stable management error for archive operations. +func (d *DataFiles) ownerRoot(ctx context.Context, id string, create bool) (*os.Root, error) { + if !manifest.ValidID(id) { + return nil, plugin.ErrInvalidArgument + } + if err := ctx.Err(); err != nil { + return nil, err + } + root, err := os.OpenRoot(d.root) + if err != nil { + return nil, err + } + defer func() { _ = root.Close() }() + info, err := root.Lstat(id) + if errors.Is(err, fs.ErrNotExist) && create { + if err := root.Mkdir(id, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, err + } + info, err = root.Lstat(id) + } + if errors.Is(err, fs.ErrNotExist) { + return nil, plugin.ErrDataNotFound + } + if err != nil { + return nil, err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil, plugin.ErrDataInvalid + } + return root.OpenRoot(id) +} diff --git a/backend/internal/plugin/provider/data_files_test.go b/backend/internal/plugin/provider/data_files_test.go new file mode 100644 index 000000000..f27d41ef6 --- /dev/null +++ b/backend/internal/plugin/provider/data_files_test.go @@ -0,0 +1,80 @@ +package provider + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/TokenFlux/TokenRouter/internal/plugin" +) + +// data_files_test.go verifies owner-directory persistence and scoped removal. +// Tests use real private directories to exercise restart discovery, symlink +// rejection, cancellation, and isolation from disposable package cleanup. + +// TestDataFilesSurvivePackageRemovalAndRestart writes business data alongside disposable state. +// Ordinary package cleanup and rebuilding the file-store adapter retain the owner's bytes and usage. +func TestDataFilesSurvivePackageRemovalAndRestart(t *testing.T) { + base := t.TempDir() + data, err := NewDataFiles(filepath.Join(base, "data")) + require.NoError(t, err) + directory, err := data.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(directory, "plugin.sqlite"), []byte("business records"), 0o600)) + packages, err := NewPackageFiles(base, DownloadOptions{}) + require.NoError(t, err) + require.NoError(t, packages.Remove(t.Context(), "test.plugin")) + restarted, err := NewDataFiles(filepath.Join(base, "data")) + require.NoError(t, err) + items, err := restarted.List(t.Context()) + require.NoError(t, err) + require.Equal(t, []plugin.DataUsage{{PluginID: "test.plugin", Bytes: 16, Files: 1}}, items) + same, err := restarted.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + require.Equal(t, directory, same) +} + +// TestDataFilesRejectEscapingOwners creates another owner's files and a linked alias. +// Invalid identifiers and linked directories cannot expose those files, while purging the alias retains its target. +func TestDataFilesRejectEscapingOwners(t *testing.T) { + data, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + other, err := data.Directory(t.Context(), "test.other") + require.NoError(t, err) + filename := filepath.Join(other, "keep") + require.NoError(t, os.WriteFile(filename, []byte("private"), 0o600)) + for _, id := range []string{"../test.other", "", "/tmp/test.plugin", "test.plugin/../test.other"} { + _, err := data.Directory(t.Context(), id) + require.ErrorIs(t, err, plugin.ErrInvalidArgument) + require.ErrorIs(t, data.Remove(t.Context(), id), plugin.ErrInvalidArgument) + } + if runtime.GOOS == "windows" { + t.Skip("creating symbolic links requires platform-specific privileges") + } + require.NoError(t, os.Symlink(other, filepath.Join(data.root, "test.alias"))) + _, err = data.Directory(t.Context(), "test.alias") + require.ErrorIs(t, err, plugin.ErrDataInvalid) + require.NoError(t, data.Remove(t.Context(), "test.alias")) + require.FileExists(t, filename) + require.NoError(t, data.Remove(t.Context(), "test.other")) + require.NoError(t, data.Remove(t.Context(), "test.other")) +} + +// TestDataFilesCancellationPreservesBusinessRecords cancels management before directory cleanup begins. +// Purging returns the context error and retains existing files for a later explicit retry. +func TestDataFilesCancellationPreservesBusinessRecords(t *testing.T) { + data, err := NewDataFiles(t.TempDir()) + require.NoError(t, err) + directory, err := data.Directory(t.Context(), "test.plugin") + require.NoError(t, err) + filename := filepath.Join(directory, "keep") + require.NoError(t, os.WriteFile(filename, []byte("retained"), 0o600)) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, data.Remove(ctx, "test.plugin"), context.Canceled) + require.FileExists(t, filename) +} diff --git a/backend/internal/plugin/provider/doc.go b/backend/internal/plugin/provider/doc.go index 7fbc22c5d..14f0eea0c 100644 --- a/backend/internal/plugin/provider/doc.go +++ b/backend/internal/plugin/provider/doc.go @@ -10,6 +10,7 @@ // - index_client.go fetches bounded signed catalogs under explicit trust policies. // - process_*.go handles child commands, platform process rules, resources, and bounded logs. // - package_*.go handles uploads, downloads, verification, and development directories. +// - data_files.go and data_archives.go retain local business files and validate stopped-owner snapshots. // - host_*.go binds capabilities, translates RPCs, observes writes, and tracks actual callback return. // - grpc_plugin.go implements the go-plugin handshake and broker service connection. package provider diff --git a/backend/internal/plugin/provider/package_repair_scenario_test.go b/backend/internal/plugin/provider/package_repair_scenario_test.go index 5311791dd..483d7e0e7 100644 --- a/backend/internal/plugin/provider/package_repair_scenario_test.go +++ b/backend/internal/plugin/provider/package_repair_scenario_test.go @@ -76,7 +76,7 @@ func TestPackageRepairRestoresSameSizeCorruption(t *testing.T) { processes := NewProcesses(plugin.RuntimeOptions{}, nil, nil) manager := plugin.NewManager( plugin.ManagerOptions{Enabled: true, MaxRunning: 1, HostVersion: "0.6.0"}, - &repairInstallationStore{}, packages, processes, plugin.NewConfiguration(nil, false), nil, + &repairInstallationStore{}, packages, processes, plugin.NewConfiguration(nil, false), nil, nil, ) // Cleanup gets its own deadline because the test context is canceled before // cleanup callbacks run, while lifecycle shutdown must still finish its work. @@ -130,7 +130,7 @@ func TestForcedSameVersionPackagesRetainIndependentRollbackBytes(t *testing.T) { require.NoError(t, err) processes := NewProcesses(plugin.RuntimeOptions{}, nil, nil) manager := plugin.NewManager(plugin.ManagerOptions{Enabled: true, MaxRunning: 1, HostVersion: "0.6.0"}, - &repairInstallationStore{}, packages, processes, plugin.NewConfiguration(nil, false), nil) + &repairInstallationStore{}, packages, processes, plugin.NewConfiguration(nil, false), nil, nil) t.Cleanup(func() { require.NoError(t, manager.Stop(context.Background())) }) service := plugin.NewService(manager, packages, nil, nil) first, err := packages.Preview(t.Context(), bytes.NewReader(packageArchive(t, "1.0.0"))) diff --git a/backend/internal/plugin/provider/process_supervisor.go b/backend/internal/plugin/provider/process_supervisor.go index 2e8002a81..9f8306728 100644 --- a/backend/internal/plugin/provider/process_supervisor.go +++ b/backend/internal/plugin/provider/process_supervisor.go @@ -564,7 +564,19 @@ func (s *supervisor) start() (_ *generation, resultErr error) { }) ctx, cancel := context.WithTimeout(s.ctx, s.options.InitTimeout) defer cancel() - _, err = conn.services.Plugin.Init(ctx, &pluginpb.InitRequest{HostVersion: s.options.HostVersion, ApiVersion: sdk.ProtocolVersion, PluginId: spec.PluginID, PluginVersion: spec.Version, PreviousVersion: spec.PreviousVersion, InstanceId: s.options.InstanceID, ConfigJson: spec.ConfigJSON, HostBrokerId: brokerID, GrantedPermissions: spec.Permissions, StateDir: spec.StateDir}) + _, err = conn.services.Plugin.Init(ctx, &pluginpb.InitRequest{ + HostVersion: s.options.HostVersion, + ApiVersion: sdk.ProtocolVersion, + PluginId: spec.PluginID, + PluginVersion: spec.Version, + PreviousVersion: spec.PreviousVersion, + InstanceId: s.options.InstanceID, + ConfigJson: spec.ConfigJSON, + HostBrokerId: brokerID, + GrantedPermissions: spec.Permissions, + StateDir: spec.StateDir, + DataDir: spec.DataDir, + }) if err != nil { return nil, err } diff --git a/backend/internal/plugin/runtime.go b/backend/internal/plugin/runtime.go index 6d27b7c33..ddba99680 100644 --- a/backend/internal/plugin/runtime.go +++ b/backend/internal/plugin/runtime.go @@ -94,15 +94,20 @@ type LogReader interface { // ProcessSpec supplies immutable inputs for starting one plugin generation. type ProcessSpec struct { - PluginID string // publisher-qualified installation identifier - Version string // desired plugin release version - PreviousVersion string // previously running release version - Executable string // absolute verified executable path - BinarySHA256 string // expected executable SHA-256 digest - StateDir string // absolute disposable process state directory - ConfigJSON []byte // normalized decrypted configuration object - Permissions []string // administrator-granted capability names - MemoryMiB int // advisory process memory limit + PluginID string // publisher-qualified installation identifier + Version string // desired plugin release version + PreviousVersion string // previously running release version + Executable string // absolute verified executable path + BinarySHA256 string // expected executable SHA-256 digest + StateDir string // absolute disposable process state directory + + // DataDir identifies the host-created persistent directory for business files. + // Internal callers without local storage leave it empty; production supplies it. + DataDir string + + ConfigJSON []byte // normalized decrypted configuration object + Permissions []string // administrator-granted capability names + MemoryMiB int // advisory process memory limit // VerifyBinary is false only for trusted development directories; installed release executables require a matching digest before process launch. VerifyBinary bool } diff --git a/backend/internal/plugin/service.go b/backend/internal/plugin/service.go index ced84f390..975c03618 100644 --- a/backend/internal/plugin/service.go +++ b/backend/internal/plugin/service.go @@ -270,11 +270,14 @@ func (s *Service) Disable(ctx context.Context, id string, actor int64) (*ChangeR // Uninstall drains admitted work before transactionally deleting installation intent and optional KV. // Drain failure leaves the installation intact with request admission closed for -// retry. Ledger history always remains, and local files are removed after commit. +// retry. Ledger history always remains; package files and disposable state are +// removed after commit. Durable business files remain until explicit data purge. // // The same plugin lock spans admission closure, callback completion, store deletion, -// and local cleanup. An arriving enable or reinstall request therefore cannot -// create a generation between drain and purge. Missing records still clean managed +// and local cleanup. A purging uninstall also removes the durable data directory +// after the host transaction commits; a failed removal remains retryable. An +// arriving enable or reinstall request cannot create a generation between drain +// and purge. Missing records still clean managed // files, supporting retries after a committed deletion whose filesystem cleanup // failed. PostgreSQL deletion owns the atomic installation and KV transaction. func (s *Service) Uninstall(ctx context.Context, id string, purge bool) error { @@ -286,6 +289,13 @@ func (s *Service) Uninstall(ctx context.Context, id string, purge bool) error { if err := s.manager.store.Delete(ctx, id, purge); err != nil { return err } + if purge && s.manager.data != nil { + // Host KV is already committed; preserve any failed filesystem cleanup + // for a later purge retry before removing the remaining package state. + if err := s.manager.data.Remove(ctx, id); err != nil { + return err + } + } return s.manager.syncLocked(ctx, id) }) } diff --git a/backend/tests/integration/plugins/catalog_scenario_integration_test.go b/backend/tests/integration/plugins/catalog_scenario_integration_test.go index a87c9c33f..31cec05aa 100644 --- a/backend/tests/integration/plugins/catalog_scenario_integration_test.go +++ b/backend/tests/integration/plugins/catalog_scenario_integration_test.go @@ -117,7 +117,7 @@ func TestSignedCatalogInstallationAndFailedUpdate(t *testing.T) { require.NoError(t, err) configuration := plugin.NewConfiguration(cipher, true) processes := provider.NewProcesses(plugin.RuntimeOptions{HostVersion: "0.6.0", InstanceID: "catalog-test", Timezone: "UTC"}, logger, nil) - manager := plugin.NewManager(plugin.ManagerOptions{Enabled: true, MaxRunning: 20, HostVersion: "0.6.0", StateRoot: filepath.Join(directory, "state")}, store, packages, processes, configuration, logger) + manager := plugin.NewManager(plugin.ManagerOptions{Enabled: true, MaxRunning: 20, HostVersion: "0.6.0", StateRoot: filepath.Join(directory, "state")}, store, packages, processes, configuration, nil, logger) catalog := plugin.NewCatalogService(plugin.CatalogOptions{Enabled: true, Host: manifest.HostInfo{Version: "0.6.0", APIVersions: []int{1}, Platform: platform, Permissions: manifest.Permissions()}}, store, store, fetcher, logger) t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) diff --git a/backend/tests/integration/plugins/installation_scenario_integration_test.go b/backend/tests/integration/plugins/installation_scenario_integration_test.go index 2e34667d8..99a380c74 100644 --- a/backend/tests/integration/plugins/installation_scenario_integration_test.go +++ b/backend/tests/integration/plugins/installation_scenario_integration_test.go @@ -55,7 +55,7 @@ func TestInstalledPluginHTTPPersistenceAndRecovery(t *testing.T) { packages, err := provider.NewPackageFiles(directory, provider.DownloadOptions{}) require.NoError(t, err) processes := provider.NewProcesses(plugin.RuntimeOptions{HostVersion: "0.5.4", InstanceID: "installation-test", Timezone: "UTC"}, logger, nil) - manager := plugin.NewManager(plugin.ManagerOptions{Enabled: true, MaxRunning: 20, HostVersion: "0.5.4", StateRoot: filepath.Join(directory, "state")}, store, packages, processes, configuration, logger) + manager := plugin.NewManager(plugin.ManagerOptions{Enabled: true, MaxRunning: 20, HostVersion: "0.5.4", StateRoot: filepath.Join(directory, "state")}, store, packages, processes, configuration, nil, logger) t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -148,7 +148,7 @@ func TestInstalledPluginHTTPPersistenceAndRecovery(t *testing.T) { // Stop the first owner before reconstructing the same durable installation; the second manager must restore enabled intent without an HTTP enable request. require.NoError(t, manager.Stop(t.Context())) restartedProcesses := provider.NewProcesses(plugin.RuntimeOptions{HostVersion: "0.5.4", InstanceID: "restarted-test", Timezone: "UTC"}, logger, nil) - restarted := plugin.NewManager(plugin.ManagerOptions{Enabled: true, MaxRunning: 20, HostVersion: "0.5.4", StateRoot: filepath.Join(directory, "state")}, store, packages, restartedProcesses, configuration, logger) + restarted := plugin.NewManager(plugin.ManagerOptions{Enabled: true, MaxRunning: 20, HostVersion: "0.5.4", StateRoot: filepath.Join(directory, "state")}, store, packages, restartedProcesses, configuration, nil, logger) t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/docs/architecture/index.md b/docs/architecture/index.md index 853c16943..5b0ff7fce 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -8,7 +8,7 @@ ## 文档 -- [插件系统](plugin_system.md)：单实例插件安装、签名索引、版本恢复、配置密钥、进程、租约、回调、写入台账、只读权限、开发目录及信任边界。读取时机：修改插件安装管理、索引市场、更新回滚、配置、启停、回调权限或开发模式时读取。 +- [插件系统](plugin_system.md)：单实例插件安装、签名索引、版本恢复、配置密钥、持久目录、进程、租约、回调、写入台账、只读权限、开发目录及信任边界。读取时机：修改插件安装管理、索引市场、更新回滚、配置、启停、回调权限、本地数据或开发模式时读取。 - [后端模块地图](backend_modules.md)：后端目录、每个包的职责、静态依赖和主要模块的协作图。读取时机：查找后端代码、判断包的归属、修改模块接口或依赖时读取。 - [前端模块与依赖](frontend_modules.md)：页面私有模块、共享组件、API 类型归属、依赖检查和启动约束。读取时机：组织前端目录、移动类型或组件、修改路由装配和跨页面依赖时读取。 diff --git a/docs/architecture/plugin_system.md b/docs/architecture/plugin_system.md index 5826292af..59290aad3 100644 --- a/docs/architecture/plugin_system.md +++ b/docs/architecture/plugin_system.md @@ -1,6 +1,6 @@ # 插件系统 -本文说明插件安装与版本管理、进程启停、请求租约、宿主能力和前端资源的运行边界。插件声明、安装包和 SDK 协议见[插件契约](../interfaces/plugin_contracts.md)，部署、备份和恢复步骤见[插件运维](../operations/plugin_operations.md)。 +本文说明插件安装与版本管理、进程启停、请求租约、持久业务文件、宿主能力和前端资源的运行边界。插件声明、安装包和 SDK 协议见[插件契约](../interfaces/plugin_contracts.md)，部署、备份和恢复步骤见[插件运维](../operations/plugin_operations.md)。 ## 章节导航 @@ -16,6 +16,7 @@ - [只读能力](#read_capabilities)：查询范围与字段限制。 - [KV 存储](#kv_storage)：条件写、配额和过期清理。 - [定时任务](#task_scheduling)：调度、取消与最近执行结果。 +- [持久业务文件](#persistent_data)：默认目录、保留清除与离线备份。 - [开发目录](#development_mode)：本地开发覆盖与持久记录。 @@ -80,7 +81,7 @@ SDK 控件复用宿主原生实现；对话框的 Teleport 面板携带插件样 `installed//sha256-` 的标记绑定归档、manifest、二进制摘要及资产 SRI。同版本的不同构建使用独立目录，强制覆盖不会改写回滚包。旧的 `installed//` 目录继续按已接受摘要读取，在不再被当前或上一包引用后清理。安装与恢复时核对全部保留文件的内容摘要，文件长度不变的损坏也会拒绝复用；普通资产读取仍只检查元数据与读取边界。索引来源缺失或损坏本地文件时按已接受的 URL 和摘要重新下载；仅允许 HTTPS，限制重定向、大小和时间，GitHub 下载前缀只适用于声明的两个 GitHub 主机。上传来源缺失或损坏文件返回 `PLUGIN_PACKAGE_MISSING`，管理员重新上传相同 ID、版本和 manifest 摘要的包进行修复。修复先排空旧进程，保留配置、权限、额度及 KV，再发布新文件身份。 -卸载先关闭请求准入并排空宿主回调，然后在同一事务里删除安装记录及可选 KV，最后删除管理目录。排空失败保留安装记录；文件清理失败可重试。台账始终保留，以便后续安装继续识别已执行的幂等键。 +卸载先关闭请求准入并排空宿主回调，然后在同一事务里删除安装记录及可选 KV，最后删除安装包和可丢弃状态。显式清除还会删除持久业务目录；文件清除失败可按同一插件 ID 重试。排空失败保留安装记录；文件清理失败可重试。台账始终保留，以便后续安装继续识别已执行的幂等键。 管理员在 `/admin/plugins` 目录页启停插件，在 `/admin/plugins/:pluginId` 详情页查看运行状态、采样资源、配置、额度、日志和版本，在 `/admin/plugins/sources` 管理索引来源。操作复用现有 TOTP step-up 开关与验证流程。全局插件关闭时保留查看与卸载入口，执行和新安装控件不可用。上传先返回未执行的包预览，随后确认完整权限集合并安装；需要配置和进程启动失败分别呈现，不能用“已安装”替代“运行中”。 @@ -158,6 +159,17 @@ Manager 持有单实例 `TaskScheduler`，按 manifest 的五段 cron 或 `@ever 禁用、重启和卸载先暂停后续调度并取消当前调用，再排空进程。暂停保留最近结果，成功卸载清除内存任务记录；宿主重启后结果也会清空。管理接口返回任务名、下一次时间、是否执行中，以及最近一次实际执行的版本、计划时间、开始时间、耗时和错误。错误文本最多保留 1024 字节；跳过的轮次不覆盖已有执行结果。 + +## 持久业务文件 + +`provider.DataFiles` 管理 `plugins.dir/data//`。Manager 在启动后端贡献前创建目录，经进程初始化的 `data_dir` 字段交给 `sdk.Env.DataDir`。路径由宿主确定，默认部署随已有数据卷持久化，插件可以直接在其中创建 SQLite 数据库和私有文件。驱动、事务、数据格式检查和 schema 升级由插件负责。 + +目录独立于安装包和 `StateDir`，重启、禁用、升级、版本回滚和普通卸载均保留文件。相同 ID 重装后读取同一目录。已卸载目录仍出现在管理清单中；显式清除会删除该 ID 的 KV 和本地目录，保留资金及投递台账。KV 删除与文件清理分别提交，文件失败会留下可发现、可重试的数据。数据库版本不兼容时，插件应在 Init 返回错误，避免回滚程序改写无法读取的数据。 + +导出和恢复沿用 Manager 的每插件锁，要求安装已禁用或不存在，并再次等待进程及宿主回调排空。ZIP 导出完成后才开始下载，临时快照独立于后续启用操作。恢复先验证插件 ID、格式版本、路径、大小、条目集合、SHA-256 和 ZIP 校验，再发布到空目录；非空目录返回冲突。档案最大 1 GiB，解压文件总量最大 1 GiB，最多 10000 个条目。超出上限的数据使用停机后的数据卷备份。 + +宿主管理操作使用受根目录限制的文件句柄，拒绝符号链接和特殊文件。插件仍属于管理员信任的同用户进程，目录分配不构成操作系统隔离。外部 PostgreSQL、MySQL 和 S3/MinIO 由插件通过自身配置连接，其备份和清理由插件或部署者管理。 + ## 开发目录 diff --git a/docs/guides/plugins/sdk_reference.md b/docs/guides/plugins/sdk_reference.md index b6bf6ed0e..fe8b13f16 100644 --- a/docs/guides/plugins/sdk_reference.md +++ b/docs/guides/plugins/sdk_reference.md @@ -6,7 +6,7 @@ Go 入口是 `sdk.Serve(plugin)`，前端入口是 `definePlugin({pages})`。完 | Go 接口 | 用途 | | --- | --- | -| `Plugin.Init(ctx, env)` | 接收插件/宿主版本、配置、宿主客户端与状态目录；成功后才可接收调用 | +| `Plugin.Init(ctx, env)` | 接收插件/宿主版本、配置、宿主客户端、缓存目录与持久数据目录；成功后才可接收调用 | | `HTTPPlugin.Handler()` | 提供 manifest 声明的 HTTP 处理器 | | `TaskPlugin.RunTask(ctx, name, scheduled)` | 处理宿主时区调度的任务；同名任务不重叠 | | `ConfigWatcher.ConfigChanged(ctx, cfg)` | 应用已保存的配置；未实现时宿主重启进程 | @@ -39,6 +39,16 @@ Config 提供 String、Float、Int、Bool、Secret 等读取器。secret 在宿 `EMAIL_DELIVERY_UNKNOWN` 表示宿主无法确定投递结果。应向管理员展示待核对状态，不换新键自动再发。其他宿主哨兵错误可用 `errors.Is` 判断；HTTP 处理器可使用 `sdk.WriteJSON` 和 `sdk.WriteError`。 +## 本地持久存储 + +`env.DataDir` 是宿主自动创建的插件专属持久目录。SQLite 插件引入自己的驱动后，将数据库路径设为 `filepath.Join(env.DataDir, "plugin.sqlite")` 即可，管理员无需配置连接地址。Init 应检查 DataDir 非空；Shutdown 应停止工作并关闭数据库连接。`env.StateDir` 用于缓存和临时文件。 + +本地目录在普通卸载后保留，重装相同 ID 时复用。管理员可在插件目录的“本地数据”页查看占用、导出 ZIP、恢复空目录或清除已卸载数据。导出前先停用插件。插件负责数据格式版本与代码回滚的兼容检查。宿主分配目录，不提供 SQLite 驱动或跨存储事务。 + +单元测试使用 `plugintest.New(t).Env`，其中的 `DataDir` 和 `StateDir` 是每个测试宿主独立的可写临时目录。插件完成测试后应关闭数据库和文件句柄，再由 Go testing 自动删除目录。该工具不模拟宿主更新、卸载、备份或恢复流程。 + +PostgreSQL/MySQL 连接串及 S3/MinIO 密钥由插件在 manifest config 中声明，敏感字段使用 secret，再通过 `env.Config.Secret` 读取。插件使用自己的客户端建立连接，并在 network 中声明外部主机名。外部存储不随本地目录导出或清除。 + ## KV 与数据兼容 Put 默认无条件写；`IfAbsent()` 只创建不存在或已过期的逻辑键，`IfVersion(v)` 使用正版本 CAS。Delete 的 `ifVersion=0` 无条件，正数要求匹配。TTL 为零时不过期；过期键逻辑上不存在，清理前重建仍会延续物理版本号。 diff --git a/docs/interfaces/plugin_contracts.md b/docs/interfaces/plugin_contracts.md index 1349c92ce..c9ac4b530 100644 --- a/docs/interfaces/plugin_contracts.md +++ b/docs/interfaces/plugin_contracts.md @@ -53,7 +53,7 @@ `/api/v1/admin/plugins` 提供已安装列表和安装操作；`/uploads` 接收 multipart 的 `file` 字段，返回 `upload_token`、过期时间、manifest、摘要和平台。上传安装使用 `{source: "upload", upload_token, accept_permissions, enable}`，索引安装使用 `{source: "index", index_id, plugin_id, version, accept_permissions, enable}`，响应包含 installation、needs_configuration 和 repaired。需要确认 manifest 中的全部权限。安装和更新都接受可选的 `ignore_host_version`、`force`，默认均为 false；前者仅跳过最低和最高网关版本要求，后者允许替换同 ID 的持久化安装，包括相同版本。安装接口启用 force 时可显式替换来源；更新接口的索引来源仍固定为当前来源。未开启 force 时，同版本上传仍只用于修复缺失或无效的已安装包。 -`/:id` 返回详情或卸载；DELETE 的 `purge_data=true` 清除 KV，缺省保留，台账始终保留。`/:id/enable`、`disable`、`restart` 控制运行意图和本地进程；`/:id/logs` 返回最近 500 行。安装列表和详情的可选 `tasks` 字段返回已登记任务的内存状态及最近执行结果；`next_run_at` 为零值时表示暂停。状态字段见[定时任务](../architecture/plugin_system.md#task_scheduling)。所有路由登记到 adminapi，执行代码、配置和额度变更通过实际 HTTP step-up guard；multipart 上传在 MCP 中使用浏览器入口。 +`/:id` 返回详情或卸载；DELETE 的 `purge_data=true` 清除 KV 与本地持久文件，缺省保留，台账始终保留。KV 与安装记录先事务提交，再清理文件；清理失败可重试。`/:id/enable`、`disable`、`restart` 控制运行意图和本地进程；`/:id/logs` 返回最近 500 行。安装列表和详情的可选 `tasks` 字段返回已登记任务的内存状态及最近执行结果；`next_run_at` 为零值时表示暂停。状态字段见[定时任务](../architecture/plugin_system.md#task_scheduling)。所有路由登记到 adminapi，执行代码、配置和额度变更通过实际 HTTP step-up guard；multipart 上传在 MCP 中使用浏览器入口。 `/:id/update` 接受 `{version, accept_permissions}` 或 `{upload_token, accept_permissions}`。索引更新只能来自当前安装的来源；上传更新可以替换当前分发来源。新增权限未确认时，`PLUGIN_PERMISSIONS_NOT_ACCEPTED` 的 metadata 同时包含完整 permissions 与 new_permissions。`ignore_host_version` 随当前及上一包快照保存，重启和回滚继续使用对应决定；新一次安装或更新缺省重新执行版本检查。安装 DTO 的同名字段返回当前决定。`force` 只作用于本次请求，不跳过完整性、协议、平台、权限确认或配置校验，也不能覆盖开发目录。`/:id/rollback` 交换完整版本快照；失败更新返回 `PLUGIN_UPDATE_FAILED`，metadata 包含 version、start_error、最多 50 行 logs，以及成功恢复时的 restored_version 或恢复失败时的 rollback_error。 @@ -63,6 +63,19 @@ 管理表单保留数值输入原文，提交前检查整数安全范围与金额的实际小数位，避免 JavaScript 舍入把无效输入变成可接受的值。文本长度按 Unicode 字符计数；RE2 pattern 只由服务器执行，字段失败通过 metadata.fields 中的结构化问题定位。停用安装可保存未填完整的必填项，启用前仍需通过完整验证。配置与额度是两个独立保存动作，界面明确报告部分成功，并在重试时跳过已经完成且未再修改的配置。 +本地数据管理接口如下，路径均位于 `/api/v1/admin/plugins`： + +| 方法和路径 | 行为 | +| --- | --- | +| `GET /data` | 返回持久目录的 plugin_id、bytes、files、installed、enabled，包含已卸载插件 | +| `DELETE /:id/data` | 清除已卸载插件的 KV 和本地目录，已安装时返回冲突 | +| `GET /:id/data/archive` | 导出已停用或已卸载插件的 ZIP 数据备份 | +| `POST /:id/data/archive` | 接收原始 `application/zip`，恢复到匹配 ID 的空目录或新目录 | + +清除、导出和恢复均要求管理员 step-up。恢复通过浏览器文件上传，不接受 MCP JSON 代替文件内容。备份成功响应不缓存，并返回附件下载头。档案和解压数据各限 1 GiB，最多 10000 个条目。相对路径最多 512 字节、20 层。ZIP 包含版本为 1 的 `manifest.json` 和 `data/` 下的文件及目录，清单记录 plugin_id、相对路径、目录标记、字节数和 SHA-256。符号链接、特殊文件、重复路径及未声明条目被拒绝。档案不包含宿主配置、KV、台账或外部存储。 + +`PLUGIN_DATA_REQUIRES_STOP` 要求先停用插件；`PLUGIN_DATA_EXISTS` 表示目标非空；`PLUGIN_DATA_NOT_FOUND` 表示没有本地目录；`PLUGIN_DATA_INVALID` 表示目录或档案校验失败；`PLUGIN_DATA_TOO_LARGE` 表示超过档案限制。普通插件 HTTP 的大小限制保持原值。 + ## HTTP 贡献与公开资源 @@ -83,6 +96,8 @@ `plugin-sdk/go/proto/tokenrouter/plugin/v1/plugin.proto` 是协议源，`make -C plugin-sdk/go proto` 用 go.mod 固定的工具生成 `pluginpb/v1`。Plugin 是必需服务，HTTPHandler 和 TaskRunner 对应可选贡献；Host 由宿主经 GRPCBroker 提供。 +`InitRequest.data_dir` 是宿主为该插件 ID 创建的持久目录，对应 Go `sdk.Env.DataDir`。`state_dir` 继续用于可丢弃缓存。旧宿主省略 `data_dir` 时字段为空；依赖本地业务文件的插件应在 Init 检查此值，并将正式支持该能力的宿主版本设为 `minHostVersion`。 + SDK 的 `Serve` 处理握手、日志、请求转换、配置更新、panic 恢复和父进程看门狗。HTTP 处理器 panic 返回 HTTP 500，生命周期 panic 转为 gRPC Internal；插件自行创建的 goroutine 需要自行处理 panic。忽略取消且仍未返回的 HTTP 或任务回调会让 SDK 健康检查失败。 宿主能力按站点、用户、钱包、兑换码、订阅、用量、订单、模型、邮件、验证码和 KV 分组。金额使用十进制字符串，扣款和本人用量调用从 `Caller` 中取得私有 grant。SDK 将 ErrorInfo reason 映射为可用 `errors.Is` 判断的错误，包括投递结果不明的 `ErrEmailDeliveryUnknown`；该结果不得通过更换幂等键盲目重发邮件。 @@ -96,7 +111,7 @@ SDK 的 `Serve` 处理握手、日志、请求转换、配置更新、panic 恢 `VerifyCaptcha` 要求 `captcha.verify`。站点未启用验证码时返回 `enabled=false, success=true`；启用状态与验证使用同一份配置。Turnstile 与阿里云保留原始 token，腾讯使用包含 `ticket`、`randstr` 的 JSON 字符串。SDK 从 `Caller` 传递原 HTTP 请求的客户端 IP。现有验证服务判为失败的挑战返回 `success=false`；配置或未分类服务错误返回 `HOST_UNAVAILABLE`。原认证入口继续执行其启动配置中的必需验证码约束。 -`plugintest` 提供进程内 gRPC 假宿主、余额和 KV，可注入协议错误。它不替代宿主的权限、额度、审计及 PostgreSQL 事务测试。签到示例使用 pending 记录固定奖励金额，再调用钱包幂等加款，最后 CAS 标记完成，允许恢复在 KV 认领后失败的加款。 +`plugintest` 提供进程内 gRPC 假宿主、余额和 KV，可注入协议错误。`plugintest.New(t)` 为每个测试宿主提供独立、可写的 `Env.StateDir` 和 `Env.DataDir`，测试结束时自动清理；插件应先关闭数据库和文件句柄。临时目录支持本地文件测试，不替代真实宿主的卸载保留、备份恢复、权限、额度、审计及 PostgreSQL 事务测试。签到示例使用 pending 记录固定奖励金额，再调用钱包幂等加款，最后 CAS 标记完成，允许恢复在 KV 认领后失败的加款。 ## KV 接口 diff --git a/docs/operations/observability_and_data_lifecycle.md b/docs/operations/observability_and_data_lifecycle.md index 51acde5a7..ee8cb9a34 100644 --- a/docs/operations/observability_and_data_lifecycle.md +++ b/docs/operations/observability_and_data_lifecycle.md @@ -137,7 +137,7 @@ Usage cleanup 是管理员手动创建的持久任务，需要提供时间范围 backup 使用 PostgreSQL dump，把产物流式写到本地或 S3 兼容存储；配置、记录、定时调度和恢复都受维护锁保护。默认的内容策略，会排除 `backupContentTableDataGroups` 里归类的 usage records、Ops logs、专项 audit history 和 runtime data，这四类都需要手动选择。提供商余额提醒的最近轮次和下次检查时间属于 `runtime_data`，配置仍保存在提供商 Extra 中；恢复时没有运行状态会在下一轮扫描重新检查。备份显示 completed，只说明已经写出了所选的内容，不代表这四类都在文件里。 -`plugin_indexes`、`plugin_installations`、`plugin_kv` 和 `plugin_ledger` 属于始终包含的业务数据。台账承载幂等执行证据，不能归入可排除的日志或 runtime_data。恢复数据库时也要恢复固定的 TOTP 加密密钥；本地安装包不在 dump 中，上传来源需要保留原归档，索引来源按已接受摘要重新下载。 +`plugin_indexes`、`plugin_installations`、`plugin_kv` 和 `plugin_ledger` 属于始终包含的业务数据。台账承载幂等执行证据，不能归入可排除的日志或 runtime_data。恢复数据库时也要恢复固定的 TOTP 加密密钥；本地安装包与 `plugins.dir/data//` 业务目录不在 dump 中，上传来源需要保留原归档，索引来源按已接受摘要重新下载。本地业务文件采用停用插件后的 ZIP 导出或停机数据卷备份，见[插件备份](plugin_operations.md#backup)。 新建备份的文件名统一为 `tr_YYYYMMDD_HHMMSS.sql.gz`，前缀固定为 `tr`。本地和 S3 存储使用这个文件名生成对象路径，下载和恢复读取备份记录中保存的名称与对象键；已有的 `sub2api` 前缀备份继续按原记录访问。 diff --git a/docs/operations/plugin_operations.md b/docs/operations/plugin_operations.md index c3dba855d..f7efd8ea5 100644 --- a/docs/operations/plugin_operations.md +++ b/docs/operations/plugin_operations.md @@ -17,7 +17,7 @@ 插件管理与调度属于单个后端进程。不要让多个后端实例同时管理同一组安装记录和目录。插件以宿主用户运行，独立进程用于隔离崩溃，不提供操作系统沙箱；前端插件也与宿主共享浏览器运行环境。只安装经过管理员审阅的代码。 -确保 `plugins.dir` 可写且持久化。默认目录来自 `DATA_DIR/plugins`；新安装文件在 `installed//sha256-/`，旧版的 `/` 目录仍可读取，可丢弃状态在 `state//`，上传暂存位于 `staging/`。宿主重启清理旧上传 token，上传预览最多保留 30 分钟。 +确保 `plugins.dir` 可写且持久化。默认目录来自 `DATA_DIR/plugins`；新安装文件在 `installed//sha256-/`，旧版的 `/` 目录仍可读取，持久业务文件在 `data//`，可丢弃状态在 `state//`，上传暂存位于 `staging/`。宿主重启清理旧上传 token，上传预览最多保留 30 分钟。 保存 secret 配置前，需要固定的 `security.secret_encryption_key`。备份和迁移时保留同一密钥，丢失密钥不能通过重新安装插件解密原值。配置来源和显式引用规则见[进程配置](../interfaces/configuration.md#configuration_sources)。 @@ -61,7 +61,7 @@ 将 `plugins.enabled: false` 后重启宿主。插件进程、调度和 KV 维护不再启动，贡献 API、UI 描述与公开资源返回 503 `PLUGINS_DISABLED`，菜单消失；管理员仍可查看和卸载安装记录。此开关是进程配置，网页上的启用开关不能覆盖它。 -普通禁用先停止接收新调用，再排空请求和宿主回调。卸载在排空完成后才事务删除安装记录及可选 KV；排空失败时保留安装记录供重试。默认卸载保留 KV，再安装相同 ID 可读取原业务数据。勾选清除数据会删除该插件的 KV，台账始终保留。 +普通禁用先停止接收新调用，再排空请求和宿主回调。卸载在排空完成后才事务删除安装记录及可选 KV；排空失败时保留安装记录供重试。默认卸载保留 KV 和 `data//`，再安装相同 ID 可读取原业务数据。勾选清除数据会删除该插件的 KV 和本地文件，台账始终保留。安装记录与 KV 先提交删除，后续文件清理失败可重试。插件目录的“本地数据”页包含已卸载目录，并支持清除残留。 ## 日志与资源 @@ -75,8 +75,20 @@ ## 备份与迁移 -数据库备份保留 plugin_indexes、plugin_installations、plugin_kv 和 plugin_ledger，即使关闭其他可选备份组也不会排除这些表。上传包无法重新下载，还需单独保留安装目录或原归档；state 目录只用于可丢弃缓存。 +数据库备份保留 plugin_indexes、plugin_installations、plugin_kv 和 plugin_ledger，即使关闭其他可选备份组也不会排除这些表。本地业务目录通过下述 ZIP 或停机数据卷备份单独保存，PostgreSQL dump 不包含这些文件。上传包无法重新下载，还需单独保留安装目录或原归档；state 目录只用于可丢弃缓存。 恢复时同时核对数据库、固定加密密钥和安装文件。不要把“数据库可恢复”当成上传二进制和插件数据迁移均已验证。插件升级应兼容保留的 KV 与台账，代码回滚不会撤销已经完成的资金、邮件或权益操作。 迁移 306 固定插件签发订阅码的天数；回退到忽略该字段的旧宿主会恢复旧兑换行为，限制见[升级说明](upgrade_notes.md)。 + +### 本地业务文件备份与恢复 + +1. 停用要备份的插件，等待操作成功。宿主会停止调度并排空进程及宿主回调；失败时先处理未结束的工作。 +2. 在插件目录的“本地数据”页导出该 ID 的 ZIP。导出与启用、卸载、恢复使用同一插件锁，档案生成完毕后才开始下载。数据库中提交过的 SQLite WAL 文件与主文件一起保存。 +3. 同一维护窗口内保存宿主 PostgreSQL、固定加密密钥以及需要保留的安装包。导出的 ZIP 只覆盖该插件本地文件，配置、KV、台账和外部对象分别保存。 +4. 恢复时保持插件停用。选择相同插件 ID 的 ZIP，目标目录必须不存在或为空。非空目录应先备份，再通过显式卸载清除处理，避免覆盖现有业务文件。 +5. 恢复验证全部路径、大小和校验值后才发布目录，随后安装或启用兼容版本。核对记录和附件后再恢复对外使用。 + +档案及解压总量各限 1 GiB，最多 10000 个文件与目录条目。不支持符号链接或特殊文件。更大的目录在停止宿主后随 `plugins.dir/data/` 数据卷备份，并记录相应的 PostgreSQL 恢复点。不要直接复制正在写入的 SQLite 主文件；后台 goroutine、数据库和文件描述符的关闭由插件 Shutdown 实现负责。 + +恢复不会自动回退插件数据库 schema，也不会执行外部数据库或 S3 的恢复。插件在 Init 检查自己的数据版本，无法兼容时应拒绝启动。宿主进程启动时清理 `.data-` 前缀的中断导出/恢复临时文件，保留所有有效插件 ID 的业务目录。 diff --git a/frontend/src/api/__tests__/admin.plugins.data.spec.ts b/frontend/src/api/__tests__/admin.plugins.data.spec.ts new file mode 100644 index 000000000..27121169c --- /dev/null +++ b/frontend/src/api/__tests__/admin.plugins.data.spec.ts @@ -0,0 +1,44 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { backupData } from '../admin/plugins' + +// admin.plugins.data.spec.ts verifies binary archive handling across policy errors. +// A Blob response retains administrator verification metadata, while successful +// archives are returned without envelope interpretation or content conversion +// by the browser API client and download controller. + +// The client fixture supplies completed responses and records status-validation settings without contacting the host. +const fixture = vi.hoisted(() => ({ get: vi.fn() })) +vi.mock('../client', () => ({ apiClient: { get: fixture.get } })) + +beforeEach(() => { + vi.resetAllMocks() +}) + +describe('plugin data downloads', () => { + // An administrator downloads a private archive before completing required TOTP verification. + // The JSON error arrives as a Blob and keeps its reason so the standard verification controller can retry the download. + it('retains the step-up reason from a binary policy response', async () => { + fixture.get.mockResolvedValue({ + status: 403, + data: new Blob( + [JSON.stringify({ reason: 'STEP_UP_REQUIRED', message: 'Verification required' })], + { type: 'application/json' }, + ), + }) + await expect(backupData('test.plugin')).rejects.toMatchObject({ + status: 403, + reason: 'STEP_UP_REQUIRED', + }) + }) + + // An authorized download returns ZIP bytes after the server creates a stopped-owner snapshot. + // The same Blob reaches the save operation, and native authentication failures remain eligible for the shared client's refresh handling. + it('preserves archive bytes and ordinary authentication error handling', async () => { + const archive = new Blob(['zip bytes'], { type: 'application/zip' }) + fixture.get.mockResolvedValue({ status: 200, data: archive }) + await expect(backupData('test.plugin')).resolves.toBe(archive) + const options = fixture.get.mock.calls[0]![1] as { validateStatus: (status: number) => boolean } + expect(options.validateStatus(401)).toBe(false) + expect(options.validateStatus(403)).toBe(true) + }) +}) diff --git a/frontend/src/api/admin/plugins.ts b/frontend/src/api/admin/plugins.ts index 59b916552..880535804 100644 --- a/frontend/src/api/admin/plugins.ts +++ b/frontend/src/api/admin/plugins.ts @@ -1,6 +1,7 @@ import { apiClient } from '../client' import type { PluginChange, + PluginDataUsage, PluginCatalogEntry, PluginIndex, PluginIndexInput, @@ -14,7 +15,8 @@ import type { PluginStatus, } from './plugins.types' -// plugins.ts sends installed-plugin administration through the shared panel client. +// plugins.ts sends installation and persistent-data administration through the +// shared panel client. // UI controllers own permission review and step-up retries. Upload and install // requests receive enough time for bounded package transfer and startup, while // configuration requests keep secret omission and clearing semantics explicit. @@ -195,3 +197,50 @@ export async function rollback(id: string): Promise { ) ).data } + +/** listData retrieves durable directories, including those retained after ordinary uninstall. */ +export async function listData(signal?: AbortSignal): Promise { + return (await apiClient.get('/admin/plugins/data', { signal })).data +} + +/** purgeData removes retained files and KV for an already uninstalled plugin. + * The caller confirms deletion before dispatch. Installed owners are rejected, + * and a filesystem cleanup failure can be retried using the same identity. + */ +export async function purgeData(id: string): Promise { + await apiClient.delete(`/admin/plugins/${encodeURIComponent(id)}/data`) +} + +/** backupData downloads a stable archive of a disabled or uninstalled owner's files. + * Expected policy errors remain JSON so the existing step-up controller can read + * their reasons; successful binary payloads retain their Blob representation. + */ +export async function backupData(id: string): Promise { + const response = await apiClient.get( + `/admin/plugins/${encodeURIComponent(id)}/data/archive`, + { + responseType: 'blob', + timeout: 300000, + validateStatus: (status) => + (status >= 200 && status < 300) || (status >= 400 && status < 500 && status !== 401), + }, + ) + if (response.status !== 200) { + // Decode policy responses before reporting them so binary download mode + // cannot hide the step-up reason that must trigger administrator verification. + const failure: Record = JSON.parse(await response.data.text()) + throw { ...failure, status: response.status } + } + return response.data +} + +/** restoreData uploads a backup for a matching disabled or uninstalled plugin ID. + * The destination must be absent or empty. The file stays owned by the caller + * so a step-up retry can resend the same bytes after verification completes. + */ +export async function restoreData(id: string, file: File): Promise { + await apiClient.post(`/admin/plugins/${encodeURIComponent(id)}/data/archive`, file, { + timeout: 300000, + headers: { 'Content-Type': 'application/zip' }, + }) +} diff --git a/frontend/src/api/admin/plugins.types.ts b/frontend/src/api/admin/plugins.types.ts index b223ba110..1daf569b4 100644 --- a/frontend/src/api/admin/plugins.types.ts +++ b/frontend/src/api/admin/plugins.types.ts @@ -238,3 +238,12 @@ export interface PluginIncompatibility { code: string // Stable host requirement failure code. detail: string // Bounded requirement failure explanation. } + +/** PluginDataUsage describes retained local business files and current installation intent. */ +export interface PluginDataUsage { + plugin_id: string // Publisher-qualified directory owner. + bytes: number // Observed regular file bytes. + files: number // Observed regular file count. + installed: boolean // An installation record currently exists. + enabled: boolean // The installation currently requests execution. +} diff --git a/frontend/src/i18n/locales/en/admin/plugins.ts b/frontend/src/i18n/locales/en/admin/plugins.ts index 54061a887..9e5e455e7 100644 --- a/frontend/src/i18n/locales/en/admin/plugins.ts +++ b/frontend/src/i18n/locales/en/admin/plugins.ts @@ -4,6 +4,23 @@ // data survives uninstalling an installed contribution. export default { + data: { + title: 'Local data', + hint: 'Business files survive ordinary uninstall. Disable a plugin before export or restore. Host database and external storage backups are separate.', + owner: 'Plugin ID', + files: 'Files', + size: 'Size', + retained: 'Uninstalled', + empty: 'No local plugin data', + export: 'Export backup', + purge: 'Delete retained data', + purgeHint: 'Delete local files and KV for {id}? This operation cannot be undone.', + restore: 'Restore backup', + restoreHint: + 'Select a ZIP exported for this plugin ID. The destination must be empty. Archives and expanded files are each limited to 1 GiB, with up to 10,000 entries.', + archive: 'Backup ZIP', + failed: 'Plugin data operation failed.', + }, updated: 'Plugin updated.', rollback: 'Roll back', rollbackTo: 'Roll back to v{version}', diff --git a/frontend/src/i18n/locales/zh/admin/plugins.ts b/frontend/src/i18n/locales/zh/admin/plugins.ts index fc807b545..ca2610569 100644 --- a/frontend/src/i18n/locales/zh/admin/plugins.ts +++ b/frontend/src/i18n/locales/zh/admin/plugins.ts @@ -4,6 +4,23 @@ // data survives uninstalling an installed contribution. export default { + data: { + title: '本地数据', + hint: '普通卸载保留业务文件。导出或恢复前请停用插件；宿主数据库和外部存储需要分别备份。', + owner: '插件 ID', + files: '文件数', + size: '大小', + retained: '已卸载', + empty: '暂无插件本地数据', + export: '导出备份', + purge: '清除保留数据', + purgeHint: '清除 {id} 的本地文件和 KV 数据？此操作无法撤销。', + restore: '恢复备份', + restoreHint: + '选择该插件 ID 导出的 ZIP 备份，目标目录须为空。压缩包与解压文件总量各限 1 GiB，最多 10,000 个条目。', + archive: 'ZIP 备份', + failed: '插件数据操作失败。', + }, updated: '插件已更新。', rollback: '回滚', rollbackTo: '回滚到 v{version}', diff --git a/frontend/src/views/admin/plugins/PluginDataPanel.vue b/frontend/src/views/admin/plugins/PluginDataPanel.vue new file mode 100644 index 000000000..237acb177 --- /dev/null +++ b/frontend/src/views/admin/plugins/PluginDataPanel.vue @@ -0,0 +1,242 @@ + + + diff --git a/frontend/src/views/admin/plugins/PluginsView.vue b/frontend/src/views/admin/plugins/PluginsView.vue index a766cbc39..5848a98b0 100644 --- a/frontend/src/views/admin/plugins/PluginsView.vue +++ b/frontend/src/views/admin/plugins/PluginsView.vue @@ -15,6 +15,7 @@ import { pluginText } from '@/components/plugin/configurationFields' import { isStepUpCancelled } from '@/composables/useStepUp' import { usePluginsStore } from '@/stores/plugins' import InstalledPluginList from './InstalledPluginList.vue' +import PluginDataPanel from './PluginDataPanel.vue' import PluginCatalogGrid from './PluginCatalogGrid.vue' import PluginUploadDialog from './PluginUploadDialog.vue' import { usePluginAdministration } from './usePluginAdministration' @@ -22,10 +23,11 @@ import { usePluginCatalog } from './usePluginCatalog' import { pluginErrorMessage } from './pluginErrors' // PluginsView.vue is the plugin directory: installed plugins, plugins with offered -// updates, and the catalog of source-qualified plugins to discover. The active view +// updates, the source-qualified catalog, and durable local plugin data. The active view // lives in query.view so links and reloads keep it, and one search box filters -// whichever view is visible. Every row and card opens the plugin detail page, which -// owns configuration, logs, versions, and destructive actions. +// installed and catalog entries. Installed rows and catalog cards open the detail +// page for configuration, logs, and versions. The local-data view also manages +// retained directories whose plugins have been uninstalled. // // Enablement stays editable from the list because it is the most frequent lifecycle // action. Writes pass through the shared administration controller, so step-up and @@ -35,7 +37,7 @@ import { pluginErrorMessage } from './pluginErrors' // controls and new package installation. /** PluginView names the query-persisted directory views. */ -type PluginView = 'installed' | 'updates' | 'discover' +type PluginView = 'installed' | 'updates' | 'discover' | 'data' // Route state, shared stores, and the two observation controllers feed the views. const { t, locale } = useI18n() @@ -45,7 +47,7 @@ const discovery = usePluginsStore() const administration = usePluginAdministration() const { installations, loading, busy, error, stepUp } = administration const view = computed(() => - route.query.view === 'updates' || route.query.view === 'discover' + route.query.view === 'updates' || route.query.view === 'discover' || route.query.view === 'data' ? route.query.view : 'installed', ) @@ -74,6 +76,7 @@ const views = computed(() => [ count: updates.value.length, }, { key: 'discover' as const, label: t('admin.plugins.directory.discover'), count: 0 }, + { key: 'data' as const, label: t('admin.plugins.data.title'), count: 0 }, ]) const categories = computed(() => [ { value: '', label: t('admin.plugins.market.allCategories') }, @@ -231,9 +234,13 @@ function installed(result: PluginChange): void { {{ t('admin.plugins.disabledNotice') }} - {{ - actionError || pluginErrorMessage(error, t('admin.plugins.loadFailed')) - }} + {{ + actionError || pluginErrorMessage(error, t('admin.plugins.loadFailed')) + }} {{ pluginErrorMessage(catalogError, t('admin.plugins.loadFailed')) }} @@ -245,7 +252,7 @@ function installed(result: PluginChange): void { }}

-
+
-