From ecb887635ecc68e884582d3da21ebaff7acab96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:50:19 +0300 Subject: [PATCH 01/55] Add v3.9 schema for fleet models, RuntimeClass, and container extras. Upgrade SQL and Sequelize models so existing 3.8 databases can grow the new tables and columns. --- .../mysql/db_migration_mysql_v3.9.0.sql | 175 ++++++++++++++++ .../postgres/db_migration_pg_v3.9.0.sql | 172 ++++++++++++++++ .../sqlite/db_migration_sqlite_v3.9.0.sql | 189 ++++++++++++++++++ src/data/models/changetracking.js | 15 ++ src/data/models/fleet-model.js | 78 ++++++++ src/data/models/fog-models.js | 10 + src/data/models/fog-runtime-classes.js | 10 + src/data/models/fog.js | 54 +++-- src/data/models/microservice-device.js | 39 ++++ src/data/models/microservice-entrypoint.js | 31 +++ src/data/models/microservice-model-item.js | 41 ++++ src/data/models/microservice-model.js | 35 ++++ src/data/models/microservice-tmpfs.js | 39 ++++ src/data/models/microservice-ulimit.js | 40 ++++ src/data/models/microservice.js | 65 ++++++ src/data/models/microservicestatus.js | 5 + src/data/models/registry.js | 15 ++ src/data/models/runtime-class.js | 32 +++ .../seeders/mysql/db_seeder_mysql_v3.9.0.sql | 12 ++ .../seeders/postgres/db_seeder_pg_v3.9.0.sql | 11 + .../sqlite/db_seeder_sqlite_v3.9.0.sql | 11 + 21 files changed, 1064 insertions(+), 15 deletions(-) create mode 100644 src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql create mode 100644 src/data/migrations/postgres/db_migration_pg_v3.9.0.sql create mode 100644 src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql create mode 100644 src/data/models/fleet-model.js create mode 100644 src/data/models/fog-models.js create mode 100644 src/data/models/fog-runtime-classes.js create mode 100644 src/data/models/microservice-device.js create mode 100644 src/data/models/microservice-entrypoint.js create mode 100644 src/data/models/microservice-model-item.js create mode 100644 src/data/models/microservice-model.js create mode 100644 src/data/models/microservice-tmpfs.js create mode 100644 src/data/models/microservice-ulimit.js create mode 100644 src/data/models/runtime-class.js create mode 100644 src/data/seeders/mysql/db_seeder_mysql_v3.9.0.sql create mode 100644 src/data/seeders/postgres/db_seeder_pg_v3.9.0.sql create mode 100644 src/data/seeders/sqlite/db_seeder_sqlite_v3.9.0.sql diff --git a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql new file mode 100644 index 00000000..2d0b6c9c --- /dev/null +++ b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql @@ -0,0 +1,175 @@ +ALTER TABLE Architectures DROP FOREIGN KEY Architectures_ibfk_2; +ALTER TABLE Architectures DROP FOREIGN KEY Architectures_ibfk_3; +ALTER TABLE Architectures DROP INDEX idx_architecture_hal_catalog_item_id; +ALTER TABLE Architectures DROP INDEX idx_architecture_bluetooth_catalog_item_id; +ALTER TABLE Architectures DROP COLUMN hal_catalog_item_id; +ALTER TABLE Architectures DROP COLUMN bluetooth_catalog_item_id; + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); + +ALTER TABLE Registries ADD COLUMN type VARCHAR(16) NOT NULL DEFAULT 'oci'; +ALTER TABLE Registries ADD COLUMN ca TEXT; +ALTER TABLE Registries ADD COLUMN insecure BOOLEAN NOT NULL DEFAULT false; + +CREATE INDEX idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_models_registry_id ON Models (registry_id); +CREATE INDEX idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +ALTER TABLE ChangeTrackings ADD COLUMN models BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN runtime_classes BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_models BOOLEAN DEFAULT false; + +ALTER TABLE Fogs DROP COLUMN device_scan_frequency; +ALTER TABLE Fogs DROP COLUMN bluetooth; +ALTER TABLE Fogs DROP COLUMN hal; + +ALTER TABLE Fogs ADD COLUMN model_status TEXT; +ALTER TABLE Fogs ADD COLUMN runtime_classes TEXT; +ALTER TABLE Fogs ADD COLUMN available_cdi_devices TEXT; +ALTER TABLE Fogs ADD COLUMN active_models BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN model_last_update BIGINT DEFAULT 0; + +ALTER TABLE Microservices ADD COLUMN run_as_group TEXT; +ALTER TABLE Microservices ADD COLUMN read_only_root_filesystem BOOLEAN DEFAULT false; +ALTER TABLE Microservices ADD COLUMN working_dir TEXT; +ALTER TABLE Microservices ADD COLUMN cpus FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_reservation FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_swap FLOAT; +ALTER TABLE Microservices ADD COLUMN shm_size FLOAT; +ALTER TABLE Microservices ADD COLUMN sysctls TEXT; + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(64) NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX idx_microservice_model_items_name ON MicroserviceModelItems (name); + +ALTER TABLE MicroserviceStatuses ADD COLUMN pod_id TEXT; + +DROP TABLE IF EXISTS HWInfos; +DROP TABLE IF EXISTS USBInfos; diff --git a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql new file mode 100644 index 00000000..0e2ab5e4 --- /dev/null +++ b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql @@ -0,0 +1,172 @@ +ALTER TABLE "Architectures" DROP COLUMN IF EXISTS hal_catalog_item_id CASCADE; +ALTER TABLE "Architectures" DROP COLUMN IF EXISTS bluetooth_catalog_item_id CASCADE; +DROP INDEX IF EXISTS idx_architecture_hal_catalog_item_id; +DROP INDEX IF EXISTS idx_architecture_bluetooth_catalog_item_id; + +DELETE FROM "Microservices" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemImages" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemInputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemOutputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue'); + +ALTER TABLE "Registries" ADD COLUMN IF NOT EXISTS type VARCHAR(16) NOT NULL DEFAULT 'oci'; +ALTER TABLE "Registries" ADD COLUMN IF NOT EXISTS ca TEXT; +ALTER TABLE "Registries" ADD COLUMN IF NOT EXISTS insecure BOOLEAN NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON "Registries" (url, type); + +CREATE TABLE IF NOT EXISTS "Models" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON "Models" (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON "Models" (name); + +CREATE TABLE IF NOT EXISTS "RuntimeClasses" ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS "FogModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES "Models" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON "FogModels" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON "FogModels" (model_uuid); + +CREATE TABLE IF NOT EXISTS "FogRuntimeClasses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES "RuntimeClasses" (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON "FogRuntimeClasses" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON "FogRuntimeClasses" (runtime_class_name); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplateVariables" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + key TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_template_id) REFERENCES "MicroserviceTemplates" (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON "MicroserviceTemplateVariables" (microservice_template_id); + +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS models BOOLEAN DEFAULT false; +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS runtime_classes BOOLEAN DEFAULT false; +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS microservice_models BOOLEAN DEFAULT false; + +ALTER TABLE "Fogs" DROP COLUMN IF EXISTS device_scan_frequency; +ALTER TABLE "Fogs" DROP COLUMN IF EXISTS bluetooth; +ALTER TABLE "Fogs" DROP COLUMN IF EXISTS hal; + +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS model_status TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS runtime_classes TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS available_cdi_devices TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS active_models BIGINT DEFAULT 0; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS model_last_update BIGINT DEFAULT 0; + +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS run_as_group TEXT; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS read_only_root_filesystem BOOLEAN DEFAULT false; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS working_dir TEXT; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS cpus DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS memory_reservation DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS memory_swap DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS shm_size DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS sysctls TEXT; + +CREATE TABLE IF NOT EXISTS "MicroserviceEntrypoints" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON "MicroserviceEntrypoints" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceDevices" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON "MicroserviceDevices" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceTmpfs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + container_path TEXT, + size DOUBLE PRECISION, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON "MicroserviceTmpfs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceUlimits" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + soft DOUBLE PRECISION, + hard DOUBLE PRECISION, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON "MicroserviceUlimits" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceModelItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Models" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON "MicroserviceModelItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON "MicroserviceModelItems" (name); + +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS pod_id TEXT DEFAULT ''; + +DROP TABLE IF EXISTS "HWInfos"; +DROP TABLE IF EXISTS "USBInfos"; diff --git a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql new file mode 100644 index 00000000..164caf65 --- /dev/null +++ b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql @@ -0,0 +1,189 @@ +DROP INDEX IF EXISTS idx_architecture_hal_catalog_item_id; +DROP INDEX IF EXISTS idx_architecture_bluetooth_catalog_item_id; +DROP INDEX IF EXISTS idx_architecture_network_catalog_item_id; + +CREATE TABLE Architectures_new ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +INSERT INTO Architectures_new (id, name, image, description, network_catalog_item_id) +SELECT id, name, image, description, network_catalog_item_id FROM Architectures; + +DROP TABLE Architectures; + +ALTER TABLE Architectures_new RENAME TO Architectures; + +CREATE INDEX idx_architecture_network_catalog_item_id ON Architectures (network_catalog_item_id); + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); + +ALTER TABLE Registries ADD COLUMN type VARCHAR(16) DEFAULT 'oci'; +ALTER TABLE Registries ADD COLUMN ca TEXT; +ALTER TABLE Registries ADD COLUMN insecure BOOLEAN DEFAULT false; + +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON Models (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +ALTER TABLE ChangeTrackings ADD COLUMN models BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN runtime_classes BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_models BOOLEAN DEFAULT false; + +ALTER TABLE Fogs DROP COLUMN device_scan_frequency; +ALTER TABLE Fogs DROP COLUMN bluetooth; +ALTER TABLE Fogs DROP COLUMN hal; + +ALTER TABLE Fogs ADD COLUMN model_status TEXT; +ALTER TABLE Fogs ADD COLUMN runtime_classes TEXT; +ALTER TABLE Fogs ADD COLUMN available_cdi_devices TEXT; +ALTER TABLE Fogs ADD COLUMN active_models BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN model_last_update BIGINT DEFAULT 0; + +ALTER TABLE Microservices ADD COLUMN run_as_group TEXT; +ALTER TABLE Microservices ADD COLUMN read_only_root_filesystem BOOLEAN DEFAULT false; +ALTER TABLE Microservices ADD COLUMN working_dir TEXT; +ALTER TABLE Microservices ADD COLUMN cpus FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_reservation FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_swap FLOAT; +ALTER TABLE Microservices ADD COLUMN shm_size FLOAT; +ALTER TABLE Microservices ADD COLUMN sysctls TEXT; + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON MicroserviceModelItems (name); + +ALTER TABLE MicroserviceStatuses ADD COLUMN pod_id TEXT DEFAULT ''; + +DROP TABLE IF EXISTS HWInfos; +DROP TABLE IF EXISTS USBInfos; diff --git a/src/data/models/changetracking.js b/src/data/models/changetracking.js index e131dfc4..e5bae7be 100644 --- a/src/data/models/changetracking.js +++ b/src/data/models/changetracking.js @@ -78,6 +78,21 @@ module.exports = (sequelize, DataTypes) => { field: 'fog_logs', defaultValue: false }, + models: { + type: DataTypes.BOOLEAN, + field: 'models', + defaultValue: false + }, + runtimeClasses: { + type: DataTypes.BOOLEAN, + field: 'runtime_classes', + defaultValue: false + }, + microserviceModels: { + type: DataTypes.BOOLEAN, + field: 'microservice_models', + defaultValue: false + }, lastUpdated: { type: DataTypes.STRING, field: 'last_updated', diff --git a/src/data/models/fleet-model.js b/src/data/models/fleet-model.js new file mode 100644 index 00000000..92bada65 --- /dev/null +++ b/src/data/models/fleet-model.js @@ -0,0 +1,78 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FleetModel = sequelize.define('FleetModel', { + uuid: { + type: DataTypes.STRING(36), + primaryKey: true, + allowNull: false, + field: 'uuid' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + field: 'name' + }, + repo: { + type: DataTypes.TEXT, + field: 'repo' + }, + revision: { + type: DataTypes.TEXT, + field: 'revision', + defaultValue: '' + }, + files: { + type: DataTypes.TEXT, + field: 'files', + defaultValue: '[]', + get () { + const value = this.getDataValue('files') + if (value == null || value === '') { + return [] + } + try { + return JSON.parse(value) + } catch (e) { + return [] + } + }, + set (value) { + if (value == null) { + this.setDataValue('files', '[]') + return + } + this.setDataValue('files', typeof value === 'string' ? value : JSON.stringify(value)) + } + }, + format: { + type: DataTypes.STRING, + field: 'format' + } + }, { + tableName: 'Models', + timestamps: true, + underscored: true + }) + + FleetModel.associate = function (models) { + FleetModel.belongsTo(models.Registry, { + foreignKey: { + name: 'registryId', + field: 'registry_id' + }, + as: 'registry', + onDelete: 'set null' + }) + + FleetModel.belongsToMany(models.Fog, { + through: models.FogModels, + as: 'fogs', + foreignKey: 'model_uuid', + otherKey: 'fog_uuid' + }) + } + + return FleetModel +} diff --git a/src/data/models/fog-models.js b/src/data/models/fog-models.js new file mode 100644 index 00000000..fdfb78d3 --- /dev/null +++ b/src/data/models/fog-models.js @@ -0,0 +1,10 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FogModels = sequelize.define('FogModels', {}, { + tableName: 'FogModels', + timestamps: false, + underscored: true + }) + return FogModels +} diff --git a/src/data/models/fog-runtime-classes.js b/src/data/models/fog-runtime-classes.js new file mode 100644 index 00000000..1f14c40d --- /dev/null +++ b/src/data/models/fog-runtime-classes.js @@ -0,0 +1,10 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FogRuntimeClasses = sequelize.define('FogRuntimeClasses', {}, { + tableName: 'FogRuntimeClasses', + timestamps: false, + underscored: true + }) + return FogRuntimeClasses +} diff --git a/src/data/models/fog.js b/src/data/models/fog.js index 303d790c..e3d73885 100644 --- a/src/data/models/fog.js +++ b/src/data/models/fog.js @@ -237,16 +237,6 @@ module.exports = (sequelize, DataTypes) => { defaultValue: '/var/log/iofog/', field: 'log_directory' }, - bluetoothEnabled: { - type: DataTypes.BOOLEAN, - defaultValue: false, - field: 'bluetooth' - }, - abstractedHardwareEnabled: { - type: DataTypes.BOOLEAN, - defaultValue: false, - field: 'hal' - }, logFileCount: { type: DataTypes.BIGINT, get () { @@ -279,11 +269,6 @@ module.exports = (sequelize, DataTypes) => { defaultValue: 20, field: 'change_frequency' }, - deviceScanFrequency: { - type: DataTypes.INTEGER, - defaultValue: 20, - field: 'device_scan_frequency' - }, tunnel: { type: DataTypes.TEXT, defaultValue: '', @@ -343,6 +328,33 @@ module.exports = (sequelize, DataTypes) => { }, field: 'volume_mount_last_update' }, + modelStatus: { + type: DataTypes.TEXT, + field: 'model_status' + }, + runtimeClasses: { + type: DataTypes.TEXT, + field: 'runtime_classes' + }, + availableCdiDevices: { + type: DataTypes.TEXT, + field: 'available_cdi_devices' + }, + activeModels: { + type: DataTypes.BIGINT, + defaultValue: 0, + get () { + return convertToInt(this.getDataValue('activeModels'), 0) + }, + field: 'active_models' + }, + modelLastUpdate: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('modelLastUpdate'), 0) + }, + field: 'model_last_update' + }, warningMessage: { type: DataTypes.TEXT, field: 'warning_message', @@ -409,6 +421,18 @@ module.exports = (sequelize, DataTypes) => { Fog.belongsToMany(models.Tags, { through: 'IofogTags', as: 'tags' }) Fog.belongsToMany(models.VolumeMount, { through: 'FogVolumeMounts', as: 'volumeMounts' }) + Fog.belongsToMany(models.FleetModel, { + through: models.FogModels, + as: 'models', + foreignKey: 'fog_uuid', + otherKey: 'model_uuid' + }) + Fog.belongsToMany(models.RuntimeClass, { + through: models.FogRuntimeClasses, + as: 'runtimeClassLinks', + foreignKey: 'fog_uuid', + otherKey: 'runtime_class_name' + }) } return Fog diff --git a/src/data/models/microservice-device.js b/src/data/models/microservice-device.js new file mode 100644 index 00000000..56ea88a4 --- /dev/null +++ b/src/data/models/microservice-device.js @@ -0,0 +1,39 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceDevice = sequelize.define('MicroserviceDevice', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + hostPath: { + type: DataTypes.TEXT, + field: 'host_path' + }, + containerPath: { + type: DataTypes.TEXT, + field: 'container_path' + }, + permissions: { + type: DataTypes.TEXT, + field: 'permissions' + } + }, { + tableName: 'MicroserviceDevices', + timestamps: false, + underscored: true + }) + MicroserviceDevice.associate = function (models) { + MicroserviceDevice.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceDevice +} diff --git a/src/data/models/microservice-entrypoint.js b/src/data/models/microservice-entrypoint.js new file mode 100644 index 00000000..9799efb1 --- /dev/null +++ b/src/data/models/microservice-entrypoint.js @@ -0,0 +1,31 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceEntrypoint = sequelize.define('MicroserviceEntrypoint', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + entrypoint: { + type: DataTypes.TEXT, + field: 'entrypoint' + } + }, { + tableName: 'MicroserviceEntrypoints', + timestamps: false, + underscored: true + }) + MicroserviceEntrypoint.associate = function (models) { + MicroserviceEntrypoint.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceEntrypoint +} diff --git a/src/data/models/microservice-model-item.js b/src/data/models/microservice-model-item.js new file mode 100644 index 00000000..018e1c5a --- /dev/null +++ b/src/data/models/microservice-model-item.js @@ -0,0 +1,41 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceModelItem = sequelize.define('MicroserviceModelItem', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + field: 'name' + } + }, { + tableName: 'MicroserviceModelItems', + timestamps: false, + underscored: true + }) + MicroserviceModelItem.associate = function (models) { + MicroserviceModelItem.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + MicroserviceModelItem.belongsTo(models.FleetModel, { + foreignKey: { + name: 'name', + field: 'name' + }, + targetKey: 'name', + as: 'model', + onDelete: 'restrict' + }) + } + return MicroserviceModelItem +} diff --git a/src/data/models/microservice-model.js b/src/data/models/microservice-model.js new file mode 100644 index 00000000..701ba057 --- /dev/null +++ b/src/data/models/microservice-model.js @@ -0,0 +1,35 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceModel = sequelize.define('MicroserviceModel', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + bindPath: { + type: DataTypes.TEXT, + field: 'bind_path' + }, + permissions: { + type: DataTypes.TEXT, + field: 'permissions' + } + }, { + tableName: 'MicroserviceModels', + timestamps: false, + underscored: true + }) + MicroserviceModel.associate = function (models) { + MicroserviceModel.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceModel +} diff --git a/src/data/models/microservice-tmpfs.js b/src/data/models/microservice-tmpfs.js new file mode 100644 index 00000000..0553f6a1 --- /dev/null +++ b/src/data/models/microservice-tmpfs.js @@ -0,0 +1,39 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceTmpfs = sequelize.define('MicroserviceTmpfs', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + containerPath: { + type: DataTypes.TEXT, + field: 'container_path' + }, + size: { + type: DataTypes.FLOAT, + field: 'size' + }, + mode: { + type: DataTypes.TEXT, + field: 'mode' + } + }, { + tableName: 'MicroserviceTmpfs', + timestamps: false, + underscored: true + }) + MicroserviceTmpfs.associate = function (models) { + MicroserviceTmpfs.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceTmpfs +} diff --git a/src/data/models/microservice-ulimit.js b/src/data/models/microservice-ulimit.js new file mode 100644 index 00000000..3f8b5c85 --- /dev/null +++ b/src/data/models/microservice-ulimit.js @@ -0,0 +1,40 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceUlimit = sequelize.define('MicroserviceUlimit', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.TEXT, + field: 'name', + allowNull: false + }, + soft: { + type: DataTypes.FLOAT, + field: 'soft' + }, + hard: { + type: DataTypes.FLOAT, + field: 'hard' + } + }, { + tableName: 'MicroserviceUlimits', + timestamps: false, + underscored: true + }) + MicroserviceUlimit.associate = function (models) { + MicroserviceUlimit.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceUlimit +} diff --git a/src/data/models/microservice.js b/src/data/models/microservice.js index 017a3821..2529eb91 100644 --- a/src/data/models/microservice.js +++ b/src/data/models/microservice.js @@ -138,6 +138,41 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.BOOLEAN, field: 'is_controller', defaultValue: false + }, + runAsGroup: { + type: DataTypes.TEXT, + field: 'run_as_group', + defaultValue: '' + }, + readOnlyRootFilesystem: { + type: DataTypes.BOOLEAN, + field: 'read_only_root_filesystem', + defaultValue: false + }, + workingDir: { + type: DataTypes.TEXT, + field: 'working_dir', + defaultValue: '' + }, + cpus: { + type: DataTypes.FLOAT, + field: 'cpus' + }, + memoryReservation: { + type: DataTypes.FLOAT, + field: 'memory_reservation' + }, + memorySwap: { + type: DataTypes.FLOAT, + field: 'memory_swap' + }, + shmSize: { + type: DataTypes.FLOAT, + field: 'shm_size' + }, + sysctls: { + type: DataTypes.TEXT, + field: 'sysctls' } }, { tableName: 'Microservices', @@ -227,6 +262,36 @@ module.exports = (sequelize, DataTypes) => { as: 'cmd' }) + Microservice.hasMany(models.MicroserviceEntrypoint, { + foreignKey: 'microservice_uuid', + as: 'entrypoint' + }) + + Microservice.hasMany(models.MicroserviceDevice, { + foreignKey: 'microservice_uuid', + as: 'devices' + }) + + Microservice.hasMany(models.MicroserviceTmpfs, { + foreignKey: 'microservice_uuid', + as: 'tmpfs' + }) + + Microservice.hasMany(models.MicroserviceUlimit, { + foreignKey: 'microservice_uuid', + as: 'ulimits' + }) + + Microservice.hasOne(models.MicroserviceModel, { + foreignKey: 'microservice_uuid', + as: 'microserviceModel' + }) + + Microservice.hasMany(models.MicroserviceModelItem, { + foreignKey: 'microservice_uuid', + as: 'modelItems' + }) + Microservice.hasMany(models.MicroserviceCdiDev, { foreignKey: 'microservice_uuid', as: 'cdiDevices' diff --git a/src/data/models/microservicestatus.js b/src/data/models/microservicestatus.js index b9f7a3cb..e6204803 100644 --- a/src/data/models/microservicestatus.js +++ b/src/data/models/microservicestatus.js @@ -51,6 +51,11 @@ module.exports = (sequelize, DataTypes) => { defaultValue: '', field: 'container_id' }, + podId: { + type: DataTypes.TEXT, + defaultValue: '', + field: 'pod_id' + }, percentage: { type: DataTypes.FLOAT, defaultValue: 0.00, diff --git a/src/data/models/registry.js b/src/data/models/registry.js index 5a721d2c..47d25c17 100644 --- a/src/data/models/registry.js +++ b/src/data/models/registry.js @@ -62,6 +62,21 @@ module.exports = (sequelize, DataTypes) => { userEmail: { type: DataTypes.TEXT, field: 'user_email' + }, + type: { + type: DataTypes.STRING(16), + field: 'type', + defaultValue: 'oci' + }, + ca: { + type: DataTypes.TEXT, + field: 'ca', + allowNull: true + }, + insecure: { + type: DataTypes.BOOLEAN, + field: 'insecure', + defaultValue: false } }, { tableName: 'Registries', diff --git a/src/data/models/runtime-class.js b/src/data/models/runtime-class.js new file mode 100644 index 00000000..94a40c29 --- /dev/null +++ b/src/data/models/runtime-class.js @@ -0,0 +1,32 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const RuntimeClass = sequelize.define('RuntimeClass', { + name: { + type: DataTypes.STRING, + primaryKey: true, + allowNull: false, + field: 'name' + }, + handler: { + type: DataTypes.STRING, + allowNull: false, + field: 'handler' + } + }, { + tableName: 'RuntimeClasses', + timestamps: false, + underscored: true + }) + + RuntimeClass.associate = function (models) { + RuntimeClass.belongsToMany(models.Fog, { + through: models.FogRuntimeClasses, + as: 'fogs', + foreignKey: 'runtime_class_name', + otherKey: 'fog_uuid' + }) + } + + return RuntimeClass +} diff --git a/src/data/seeders/mysql/db_seeder_mysql_v3.9.0.sql b/src/data/seeders/mysql/db_seeder_mysql_v3.9.0.sql new file mode 100644 index 00000000..a52bca80 --- /dev/null +++ b/src/data/seeders/mysql/db_seeder_mysql_v3.9.0.sql @@ -0,0 +1,12 @@ +INSERT INTO Registries (url, is_public, user_name, password, user_email, type, ca, insecure) +SELECT 'https://huggingface.co', true, '', '', '', 'hf', NULL, false +FROM DUAL +WHERE NOT EXISTS ( + SELECT 1 FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf' +); + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); diff --git a/src/data/seeders/postgres/db_seeder_pg_v3.9.0.sql b/src/data/seeders/postgres/db_seeder_pg_v3.9.0.sql new file mode 100644 index 00000000..d9d13de3 --- /dev/null +++ b/src/data/seeders/postgres/db_seeder_pg_v3.9.0.sql @@ -0,0 +1,11 @@ +INSERT INTO "Registries" (url, is_public, user_name, password, user_email, type, ca, insecure) +SELECT 'https://huggingface.co', true, '', '', '', 'hf', NULL, false +WHERE NOT EXISTS ( + SELECT 1 FROM "Registries" WHERE url = 'https://huggingface.co' AND type = 'hf' +); + +DELETE FROM "Microservices" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemImages" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemInputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemOutputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue'); diff --git a/src/data/seeders/sqlite/db_seeder_sqlite_v3.9.0.sql b/src/data/seeders/sqlite/db_seeder_sqlite_v3.9.0.sql new file mode 100644 index 00000000..f212e2cf --- /dev/null +++ b/src/data/seeders/sqlite/db_seeder_sqlite_v3.9.0.sql @@ -0,0 +1,11 @@ +INSERT INTO Registries (url, is_public, user_name, password, user_email, type, ca, insecure) +SELECT 'https://huggingface.co', true, '', '', '', 'hf', NULL, false +WHERE NOT EXISTS ( + SELECT 1 FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf' +); + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); From b28e0f11a7cfc30e86b1216e55d8144cfa177511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:50:24 +0300 Subject: [PATCH 02/55] Land empty databases on the v3.9 baseline schema. Fresh installs skip the 3.8 chain; 3.8 databases still take the incremental 3.9 path only. --- src/data/migrations/README.md | 30 +- .../db_migration_mysql_baseline_v3.9.0.sql | 1358 +++++++++++++++++ .../db_migration_pg_baseline_v3.9.0.sql | 1345 ++++++++++++++++ .../db_migration_sqlite_baseline_v3.9.0.sql | 1344 ++++++++++++++++ src/data/models/index.js | 9 +- src/data/providers/database-provider.js | 738 +++++---- src/data/providers/sqlite.js | 4 +- .../mysql/db_seeder_mysql_baseline_v3.9.0.sql | 69 + .../postgres/db_seeder_pg_baseline_v3.9.0.sql | 73 + .../db_seeder_sqlite_baseline_v3.9.0.sql | 65 + .../data/schema-version-chain-sqlite.test.js | 382 +++++ .../first-fog-reconcile-sqlite.test.js | 4 +- test/support/first-fog-sqlite-harness.js | 14 +- 13 files changed, 5134 insertions(+), 301 deletions(-) create mode 100644 src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql create mode 100644 src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql create mode 100644 src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql create mode 100644 src/data/seeders/mysql/db_seeder_mysql_baseline_v3.9.0.sql create mode 100644 src/data/seeders/postgres/db_seeder_pg_baseline_v3.9.0.sql create mode 100644 src/data/seeders/sqlite/db_seeder_sqlite_baseline_v3.9.0.sql create mode 100644 test/src/data/schema-version-chain-sqlite.test.js diff --git a/src/data/migrations/README.md b/src/data/migrations/README.md index 8347fb78..193bd2a0 100644 --- a/src/data/migrations/README.md +++ b/src/data/migrations/README.md @@ -1,12 +1,32 @@ # Database migrations -## Fresh install (Controller v3.8+) +## Fresh install (empty database) -New installations use **`db_migration_*_v3.8.0.sql`** and **`db_seeder_*_v3.8.0.sql`** -(sqlite, mysql, postgres). The migration runner in `src/data/providers/database-provider.js` -records schema version **`3.8.0`** (includes embedded auth tables). +New installations apply **baseline 3.9.0** schema and seed files only. The runner in +`src/data/providers/database-provider.js` uses this path when `schema_version` is unset +and `untilVersion` is not provided. `schema_version` and `seeder_version` are then recorded +as `3.9.0`. Baseline files do not create hardware-inventory tables, architecture HAL/BLE +foreign keys, or HAL/RESTBlue catalog rows. -Greenfield only: wipe the data directory or database before upgrading from pre-3.8 builds. +| Version | Migration | Seeder | +|---------|-----------|--------| +| 3.9.0 (fresh) | `db_migration_*_baseline_v3.9.0.sql` | `db_seeder_*_baseline_v3.9.0.sql` | + +## Upgrade from 3.8.0 + +Existing 3.8.0 databases apply **incremental** 3.9.0 migrate-then-seed only. Do **not** +edit `db_*_v3.8.0.sql` files. The incremental 3.9 migration drops architecture HAL/BLE +foreign keys before deleting HAL/RESTBlue catalog rows. + +`untilVersion: '3.8.0'` still builds a 3.8 fixture via the incremental chain (used by tests). +Never apply baseline 3.9 and the 3.8 chain on the same database. + +| Version | Migration | Seeder | +|---------|-----------|--------| +| 3.8.0 | `db_migration_*_v3.8.0.sql` | `db_seeder_*_v3.8.0.sql` | +| 3.9.0 (upgrade) | `db_migration_*_v3.9.0.sql` | `db_seeder_*_v3.9.0.sql` | + +Greenfield only vs pre-3.8: wipe the data directory or database before upgrading from pre-3.8 builds. There is no v3.7→v3.8 incremental migrator. ## Pre-3.8 historical files diff --git a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql new file mode 100644 index 00000000..512e1485 --- /dev/null +++ b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql @@ -0,0 +1,1358 @@ +START TRANSACTION; + +CREATE TABLE IF NOT EXISTS Applications ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255) DEFAULT '', + is_activated BOOLEAN DEFAULT false, + is_system BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + nats_access BOOLEAN DEFAULT false, + nats_rule_id INTEGER +); + +CREATE TABLE IF NOT EXISTS Registries ( + id INT AUTO_INCREMENT PRIMARY KEY, + url VARCHAR(255), + is_public BOOLEAN, + user_name TEXT, + password TEXT, + user_email TEXT, + type VARCHAR(16) NOT NULL DEFAULT 'oci', + ca TEXT, + insecure BOOLEAN NOT NULL DEFAULT false +); + +CREATE TABLE IF NOT EXISTS CatalogItems ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255), + category TEXT, + config_example VARCHAR(255) DEFAULT '{}', + publisher TEXT, + disk_required BIGINT DEFAULT 0, + ram_required BIGINT DEFAULT 0, + picture VARCHAR(255) DEFAULT 'images/shared/default.png', + is_public BOOLEAN DEFAULT false, + registry_id INT, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_catalog_item_registry_id ON CatalogItems (registry_id); + +CREATE TABLE IF NOT EXISTS Architectures ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_architecture_network_catalog_item_id ON Architectures (network_catalog_item_id); + +CREATE TABLE IF NOT EXISTS Fogs ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) DEFAULT 'Unnamed ioFog 1', + location TEXT, + gps_mode TEXT, + latitude FLOAT, + longitude FLOAT, + description TEXT, + last_active BIGINT, + daemon_status VARCHAR(36) DEFAULT 'NOT_PROVISIONED', + daemon_operating_duration BIGINT DEFAULT 0, + daemon_last_start BIGINT, + memory_usage FLOAT DEFAULT 0.000, + disk_usage FLOAT DEFAULT 0.000, + cpu_usage FLOAT DEFAULT 0.00, + memory_violation TEXT, + disk_violation TEXT, + cpu_violation TEXT, + system_available_disk BIGINT, + system_available_memory BIGINT, + system_total_cpu FLOAT, + security_status VARCHAR(36) DEFAULT 'OK', + security_violation_info VARCHAR(36) DEFAULT 'No violation', + catalog_item_status TEXT, + repository_count BIGINT DEFAULT 0, + repository_status TEXT, + system_time BIGINT, + last_status_time BIGINT, + ip_address VARCHAR(36) DEFAULT '0.0.0.0', + ip_address_external VARCHAR(36) DEFAULT '0.0.0.0', + host VARCHAR(36), + catalog_item_message_counts TEXT, + last_command_time BIGINT, + network_interface VARCHAR(36) DEFAULT 'dynamic', + docker_url VARCHAR(255) DEFAULT 'unix:///run/edgelet/contaienrd.sock', + disk_limit FLOAT DEFAULT 50, + disk_directory VARCHAR(255) DEFAULT '/var/lib/iofog-agent/', + memory_limit FLOAT DEFAULT 4096, + cpu_limit FLOAT DEFAULT 80, + log_limit FLOAT DEFAULT 10, + log_directory VARCHAR(255) DEFAULT '/var/log/iofog/', + log_file_count BIGINT DEFAULT 10, + `version` TEXT, + is_ready_to_upgrade BOOLEAN DEFAULT TRUE, + is_ready_to_rollback BOOLEAN DEFAULT FALSE, + status_frequency INT DEFAULT 10, + change_frequency INT DEFAULT 20, + tunnel VARCHAR(255) DEFAULT '', + isolated_docker_container BOOLEAN DEFAULT FALSE, + docker_pruning_freq INT DEFAULT 0, + available_disk_threshold FLOAT DEFAULT 20, + log_level VARCHAR(10) DEFAULT 'INFO', + is_system BOOLEAN DEFAULT FALSE, + router_id INT DEFAULT 0, + time_zone VARCHAR(36) DEFAULT 'Etc/UTC', + created_at DATETIME, + updated_at DATETIME, + arch_id INT DEFAULT 0, + container_engine VARCHAR(36), + deployment_type VARCHAR(36), + active_volume_mounts BIGINT DEFAULT 0, + volume_mount_last_update BIGINT DEFAULT 0, + warning_message TEXT, + gps_device VARCHAR(36), + gps_scan_frequency INT DEFAULT 60, + edge_guard_frequency INT DEFAULT 0, + gps_status VARCHAR(32), + nats_id INT, + available_runtimes TEXT, + model_status TEXT, + runtime_classes TEXT, + available_cdi_devices TEXT, + active_models BIGINT DEFAULT 0, + model_last_update BIGINT DEFAULT 0, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) +); + +CREATE INDEX idx_fog_arch_id ON Fogs (arch_id); + +CREATE TABLE IF NOT EXISTS ChangeTrackings ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_config BOOLEAN DEFAULT false, + reboot BOOLEAN DEFAULT false, + deletenode BOOLEAN DEFAULT false, + version BOOLEAN DEFAULT false, + microservice_list BOOLEAN DEFAULT false, + config BOOLEAN DEFAULT false, + registries BOOLEAN DEFAULT false, + tunnel BOOLEAN DEFAULT false, + router_changed BOOLEAN DEFAULT false, + prune BOOLEAN DEFAULT false, + last_updated VARCHAR(255) DEFAULT false, + iofog_uuid VARCHAR(36), + volume_mounts BOOLEAN DEFAULT false, + exec_sessions BOOLEAN DEFAULT false, + microservice_logs BOOLEAN DEFAULT false, + fog_logs BOOLEAN DEFAULT false, + models BOOLEAN DEFAULT false, + runtime_classes BOOLEAN DEFAULT false, + microservice_models BOOLEAN DEFAULT false, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_change_tracking_iofog_uuid ON ChangeTrackings (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogProvisionKeys ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + provisioning_string VARCHAR(100), + expiration_time BIGINT, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_provision_keys_iofogUuid ON FogProvisionKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogVersionCommands ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + version_command VARCHAR(100), + semver VARCHAR(100), + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_version_commands_iofogUuid ON FogVersionCommands (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Tunnels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + username TEXT, + password TEXT, + host TEXT, + remote_port INT, + local_port INT DEFAULT 22, + rsa_key TEXT, + closed BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_tunnels_iofogUuid ON Tunnels (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Microservices ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + config TEXT, + name VARCHAR(255) DEFAULT 'New Microservice', + config_last_updated BIGINT, + rebuild BOOLEAN DEFAULT false, + log_size BIGINT DEFAULT 0, + `delete` BOOLEAN DEFAULT false, + delete_with_cleanup BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + catalog_item_id INT, + registry_id INT DEFAULT 1, + iofog_uuid VARCHAR(36), + application_id INT, + run_as_user TEXT, + platform TEXT, + runtime TEXT, + annotations TEXT, + pid_mode VARCHAR(36), + ipc_mode VARCHAR(36), + schedule INT DEFAULT 50, + cpu_set_cpus TEXT, + memory_limit FLOAT, + is_activated BOOLEAN DEFAULT true, + host_network_mode BOOLEAN DEFAULT false, + is_privileged BOOLEAN DEFAULT false, + nats_rule_id INT, + nats_access BOOLEAN DEFAULT false, + nats_account_id INT, + nats_user_id INT, + nats_creds_secret_name TEXT, + is_controller BOOLEAN DEFAULT false, + run_as_group TEXT, + read_only_root_filesystem BOOLEAN DEFAULT false, + working_dir TEXT, + cpus FLOAT, + memory_reservation FLOAT, + memory_swap FLOAT, + shm_size FLOAT, + sysctls TEXT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservices_catalogItemId ON Microservices (catalog_item_id); +CREATE INDEX idx_microservices_registryId ON Microservices (registry_id); +CREATE INDEX idx_microservices_iofogUuid ON Microservices (iofog_uuid); +CREATE INDEX idx_microservices_applicationId ON Microservices (application_id); + +CREATE TABLE IF NOT EXISTS MicroserviceArgs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cmd TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_args_microserviceUuid ON MicroserviceArgs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceEnvs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + `key` TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + value_from_secret TEXT, + value_from_config_map TEXT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_envs_microserviceUuid ON MicroserviceEnvs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExtraHost ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + template_type TEXT, + name TEXT, + template TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + target_microservice_uuid VARCHAR(36), + target_fog_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_extra_host_microserviceUuid ON MicroserviceExtraHost (microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetMicroserviceUuid ON MicroserviceExtraHost (target_microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetFogUuid ON MicroserviceExtraHost (target_fog_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePorts ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + port_internal INT, + port_external INT, + is_udp BOOLEAN, + created_at DATETIME, + updated_at DATETIME, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_port_microserviceUuid ON MicroservicePorts (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'QUEUED', + operating_duration BIGINT DEFAULT 0, + start_time BIGINT DEFAULT 0, + cpu_usage FLOAT DEFAULT 0.000, + memory_usage BIGINT DEFAULT 0, + container_id VARCHAR(255) DEFAULT '', + percentage FLOAT DEFAULT 0.00, + error_message TEXT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + ip_address TEXT, + exec_session_ids TEXT, + health_status TEXT, + pod_id TEXT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_status_microserviceUuid ON MicroserviceStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS VolumeMappings ( + uuid INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + host_destination TEXT, + container_destination TEXT, + access_mode TEXT, + type TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mappings_microserviceUuid ON VolumeMappings (microservice_uuid); + +CREATE TABLE IF NOT EXISTS CatalogItemImages ( + id INT AUTO_INCREMENT PRIMARY KEY, + container_image TEXT, + catalog_item_id INT, + microservice_uuid VARCHAR(36), + arch_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_image_catalog_item_id ON CatalogItemImages (catalog_item_id); +CREATE INDEX idx_catalog_item_image_microservice_uuid ON CatalogItemImages (microservice_uuid); +CREATE INDEX idx_catalog_item_image_arch_id ON CatalogItemImages (arch_id); + +CREATE TABLE IF NOT EXISTS CatalogItemInputTypes ( + id INT AUTO_INCREMENT PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_input_type_catalog_item_id ON CatalogItemInputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS CatalogItemOutputTypes ( + id INT AUTO_INCREMENT PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_output_type_catalog_item_id ON CatalogItemOutputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS Routers ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + is_edge BOOLEAN DEFAULT true, + messaging_port INT DEFAULT 5671, + edge_router_port INT, + inter_router_port INT, + host TEXT, + is_default BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_router_iofogUuid ON Routers (iofog_uuid); + +CREATE TABLE RouterConnections ( + id INT AUTO_INCREMENT PRIMARY KEY, + source_router INT, + dest_router INT, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL, + FOREIGN KEY (source_router) REFERENCES Routers(id) ON DELETE CASCADE, + FOREIGN KEY (dest_router) REFERENCES Routers(id) ON DELETE CASCADE +); + +CREATE INDEX idx_routerconnections_sourceRouter ON RouterConnections (source_router); +CREATE INDEX idx_routerconnections_destRouter ON RouterConnections (dest_router); + +CREATE TABLE IF NOT EXISTS Config ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + `key` VARCHAR(255) NOT NULL UNIQUE, + value VARCHAR(255) NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_config_key ON Config (`key`); + +CREATE TABLE IF NOT EXISTS Tags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + value VARCHAR(255) UNIQUE NOT NULL +); + +CREATE TABLE IF NOT EXISTS IofogTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_iofogtags_fog_uuid ON IofogTags (fog_uuid); +CREATE INDEX idx_iofogtags_tag_id ON IofogTags (tag_id); + +CREATE TABLE IF NOT EXISTS ApplicationTemplates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL DEFAULT 'new-application', + description VARCHAR(255) DEFAULT '', + schema_version VARCHAR(255) DEFAULT '', + application_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS ApplicationTemplateVariables ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + application_template_id INT NOT NULL, + `key` TEXT, + description VARCHAR(255) DEFAULT '', + default_value VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (application_template_id) REFERENCES ApplicationTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_applicationtemplatevariables_application_template_id ON ApplicationTemplateVariables (application_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCdiDevices ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cdi_devices TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_cdiDevices_microserviceUuid ON MicroserviceCdiDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePubTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceSubTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicepubtags_microservice_uuid ON MicroservicePubTags (microservice_uuid); +CREATE INDEX idx_microservicesubtags_microservice_uuid ON MicroserviceSubTags (microservice_uuid); +CREATE INDEX idx_microservicepubtags_tag_id ON MicroservicePubTags (tag_id); +CREATE INDEX idx_microservicesubtags_tag_id ON MicroserviceSubTags (tag_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCapAdd ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cap_add TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capAdd_microserviceUuid ON MicroserviceCapAdd (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceCapDrop ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cap_drop TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capDrop_microserviceUuid ON MicroserviceCapDrop (microservice_uuid); + +CREATE TABLE IF NOT EXISTS FogPublicKeys ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + public_key TEXT, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_public_keys_iofogUuid ON FogPublicKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogUsedTokens ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + jti VARCHAR(255) NOT NULL, + iofog_uuid VARCHAR(36), + expiry_time BIGINT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_used_tokens_iofogUuid ON FogUsedTokens (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Secrets ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('Opaque', 'tls')), + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_secrets_name ON Secrets (name); + +CREATE TABLE IF NOT EXISTS Certificates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + subject TEXT NOT NULL, + is_ca BOOLEAN DEFAULT false, + signed_by_id INT, + hosts TEXT, + valid_from DATETIME NOT NULL, + valid_to DATETIME NOT NULL, + serial_number TEXT NOT NULL, + secret_id INT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (signed_by_id) REFERENCES Certificates (id) ON DELETE SET NULL, + FOREIGN KEY (secret_id) REFERENCES Secrets (id) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX idx_certificates_name_unique ON Certificates (name(255)); +CREATE INDEX idx_certificates_valid_to ON Certificates (valid_to); +CREATE INDEX idx_certificates_is_ca ON Certificates (is_ca); +CREATE INDEX idx_certificates_signed_by_id ON Certificates (signed_by_id); +CREATE INDEX idx_certificates_secret_id ON Certificates (secret_id); + +CREATE TABLE IF NOT EXISTS Services ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL, + resource TEXT NOT NULL, + target_port INT NOT NULL, + service_port INT, + k8s_type TEXT, + bridge_port INT, + default_bridge TEXT, + service_endpoint TEXT, + created_at DATETIME, + updated_at DATETIME, + provisioning_status VARCHAR(36) DEFAULT 'pending', + provisioning_error TEXT +); + +CREATE INDEX idx_services_name ON Services (name); +CREATE INDEX idx_services_id ON Services (id); + +CREATE TABLE IF NOT EXISTS ServiceTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + service_id INT NOT NULL, + tag_id INT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (service_id) REFERENCES Services (id) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_service_tags_service_id ON ServiceTags (service_id); +CREATE INDEX idx_service_tags_tag_id ON ServiceTags (tag_id); + +CREATE TABLE IF NOT EXISTS ConfigMaps ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + immutable BOOLEAN DEFAULT false, + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + use_vault BOOLEAN DEFAULT true +); + +CREATE INDEX idx_config_maps_name ON ConfigMaps (name); + +CREATE TABLE IF NOT EXISTS VolumeMounts ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + config_map_name VARCHAR(255), + secret_name VARCHAR(255), + version INT DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (config_map_name) REFERENCES ConfigMaps (name) ON DELETE CASCADE, + FOREIGN KEY (secret_name) REFERENCES Secrets (name) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mounts_uuid ON VolumeMounts (uuid); +CREATE INDEX idx_volume_mounts_config_map_name ON VolumeMounts (config_map_name); +CREATE INDEX idx_volume_mounts_secret_name ON VolumeMounts (secret_name); + +CREATE TABLE IF NOT EXISTS FogVolumeMounts ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + volume_mount_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (volume_mount_uuid) REFERENCES VolumeMounts (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_volume_mounts_fog_uuid ON FogVolumeMounts (fog_uuid); +CREATE INDEX idx_fog_volume_mounts_volume_mount_uuid ON FogVolumeMounts (volume_mount_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExecStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'INACTIVE', + exec_session_id VARCHAR(255), + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_status_microservice_uuid ON MicroserviceExecStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceHealthChecks ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + test TEXT, + `interval` BIGINT, + timeout BIGINT, + start_period BIGINT, + start_interval BIGINT, + retries INT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_health_check_microservice_uuid ON MicroserviceHealthChecks (microservice_uuid); + +CREATE TABLE IF NOT EXISTS Events ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + timestamp BIGINT NOT NULL, + event_type VARCHAR(20) NOT NULL, + endpoint_type VARCHAR(10) NOT NULL, + actor_id VARCHAR(255), + method VARCHAR(10), + resource_type VARCHAR(50), + resource_id VARCHAR(255), + endpoint_path TEXT NOT NULL, + ip_address VARCHAR(45), + status VARCHAR(20) NOT NULL, + status_code INT, + status_message TEXT, + request_id VARCHAR(255), + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_events_timestamp ON Events (timestamp); +CREATE INDEX idx_events_endpoint_type ON Events (endpoint_type); +CREATE INDEX idx_events_actor_id ON Events (actor_id); +CREATE INDEX idx_events_resource_type ON Events (resource_type); +CREATE INDEX idx_events_status ON Events (status); +CREATE INDEX idx_events_method ON Events (method); +CREATE INDEX idx_events_event_type ON Events (event_type); +CREATE INDEX idx_events_created_at ON Events (created_at); + +CREATE TABLE IF NOT EXISTS MicroserviceLogStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_log_status_microservice_uuid ON MicroserviceLogStatuses (microservice_uuid); +CREATE INDEX idx_microservice_log_status_session_id ON MicroserviceLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS MicroserviceExecSessions ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON MicroserviceExecSessions (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON MicroserviceExecSessions (session_id); + +CREATE TABLE IF NOT EXISTS FogLogStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + iofog_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_log_status_iofog_uuid ON FogLogStatuses (iofog_uuid); +CREATE INDEX idx_fog_log_status_session_id ON FogLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS RbacRoles ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + kind TEXT, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY unique_name (name(255)) +); + +CREATE TABLE IF NOT EXISTS RbacRoleRules ( + id INT AUTO_INCREMENT PRIMARY KEY, + role_id INT NOT NULL, + api_groups TEXT NOT NULL, + resources TEXT NOT NULL, + verbs TEXT NOT NULL, + resource_names TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (role_id) REFERENCES RbacRoles (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS RbacRoleBindings ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + kind TEXT, + role_ref TEXT NOT NULL, + subjects TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY unique_name (name(255)), + role_id INTEGER +); + +CREATE TABLE IF NOT EXISTS RbacServiceAccounts ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + role_ref TEXT, + role_id INT, + microservice_uuid VARCHAR(36), + application_id INT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (role_id) REFERENCES RbacRoles (id), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE SET NULL +); + +CREATE INDEX idx_rbac_role_rules_role_id ON RbacRoleRules (role_id); +CREATE INDEX idx_rbac_roles_name ON RbacRoles (name(255)); +CREATE INDEX idx_rbac_role_bindings_name ON RbacRoleBindings (name(255)); +CREATE INDEX idx_rbac_service_accounts_name ON RbacServiceAccounts (name(255)); + +CREATE TABLE IF NOT EXISTS RbacCacheVersion ( + id INT PRIMARY KEY DEFAULT 1, + version BIGINT NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + CONSTRAINT single_row CHECK (id = 1) +); + +CREATE INDEX idx_rbac_role_bindings_role_id ON RbacRoleBindings (role_id); + +CREATE INDEX idx_rbac_service_accounts_role_id ON RbacServiceAccounts (role_id); +CREATE UNIQUE INDEX idx_rbac_service_accounts_microservice_uuid_unique ON RbacServiceAccounts (microservice_uuid); +CREATE UNIQUE INDEX idx_rbac_service_accounts_application_id_name_unique ON RbacServiceAccounts (application_id, name(255)); + +CREATE TABLE IF NOT EXISTS ClusterControllers ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + host VARCHAR(255), + process_id INT, + last_heartbeat DATETIME, + is_active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_cluster_controllers_uuid ON ClusterControllers (uuid); +CREATE INDEX idx_cluster_controllers_host ON ClusterControllers (host); +CREATE INDEX idx_cluster_controllers_active ON ClusterControllers (is_active, last_heartbeat); + +CREATE TABLE IF NOT EXISTS NatsOperators ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY nats_operators_name (name(255)) +); + +CREATE TABLE IF NOT EXISTS NatsAccounts ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + is_system BOOLEAN DEFAULT false, + is_leaf_system BOOLEAN DEFAULT false, + operator_id INT NOT NULL, + application_id INT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (operator_id) REFERENCES NatsOperators (id) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsUsers ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + creds_secret_name TEXT NOT NULL, + is_bearer BOOLEAN DEFAULT false, + account_id INT NOT NULL, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + nats_user_rule_id INTEGER, + FOREIGN KEY (account_id) REFERENCES NatsAccounts (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS NatsInstances ( + id INT AUTO_INCREMENT PRIMARY KEY, + iofog_uuid VARCHAR(36), + is_leaf BOOLEAN DEFAULT true, + is_hub BOOLEAN DEFAULT false, + host TEXT, + server_port INT, + leaf_port INT, + cluster_port INT, + mqtt_port INT, + http_port INT, + configmap_name TEXT, + jwt_dir_mount_name TEXT, + cert_secret_name TEXT, + js_storage_size TEXT, + js_memory_store_size TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsConnections ( + id INT AUTO_INCREMENT PRIMARY KEY, + source_nats INT NOT NULL, + dest_nats INT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (source_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE, + FOREIGN KEY (dest_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsReconcileTasks ( + id INT AUTO_INCREMENT PRIMARY KEY, + reason VARCHAR(64) NOT NULL, + application_id INT, + account_rule_id INT, + user_rule_id INT, + fog_uuids TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsAccountRules ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT, + info_url TEXT, + max_connections INT, + max_leaf_node_connections INT, + max_data BIGINT, + max_exports INT, + max_imports INT, + max_msg_payload INT, + max_subscriptions INT, + exports_allow_wildcards BOOLEAN DEFAULT true, + disallow_bearer BOOLEAN, + response_permissions TEXT, + resp_max INT, + resp_ttl BIGINT, + imports TEXT, + exports TEXT, + mem_storage BIGINT, + disk_storage BIGINT, + streams INT, + consumer INT, + max_ack_pending INT, + mem_max_stream_bytes BIGINT, + disk_max_stream_bytes BIGINT, + max_bytes_required BOOLEAN, + tiered_limits TEXT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsUserRules ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT, + max_subscriptions INT, + max_payload INT, + max_data BIGINT, + bearer_token BOOLEAN DEFAULT false, + proxy_required BOOLEAN, + allowed_connection_types TEXT, + src TEXT, + times TEXT, + times_location TEXT, + resp_max INT, + resp_ttl BIGINT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + tags TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE UNIQUE INDEX idx_nats_accounts_application_id_unique ON NatsAccounts (application_id); +CREATE INDEX idx_nats_accounts_application_id ON NatsAccounts (application_id); +CREATE UNIQUE INDEX idx_nats_users_account_id_name ON NatsUsers (account_id, name(255)); +CREATE INDEX idx_nats_users_account_id ON NatsUsers (account_id); +CREATE INDEX idx_nats_users_microservice_uuid ON NatsUsers (microservice_uuid); +CREATE INDEX idx_nats_users_nats_user_rule_id ON NatsUsers (nats_user_rule_id); +CREATE UNIQUE INDEX idx_nats_instances_iofog_uuid_unique ON NatsInstances (iofog_uuid); +CREATE INDEX idx_nats_instances_iofog_uuid ON NatsInstances (iofog_uuid); +CREATE UNIQUE INDEX idx_nats_connections_source_dest_unique ON NatsConnections (source_nats, dest_nats); +CREATE INDEX idx_nats_connections_source_nats ON NatsConnections (source_nats); +CREATE INDEX idx_nats_connections_dest_nats ON NatsConnections (dest_nats); +CREATE INDEX idx_nats_account_rules_name ON NatsAccountRules (name); +CREATE INDEX idx_nats_user_rules_name ON NatsUserRules (name); +CREATE INDEX idx_nats_reconcile_tasks_status_claimed ON NatsReconcileTasks (status, claimed_at); + +CREATE INDEX idx_applications_nats_rule_id ON Applications (nats_rule_id); +CREATE INDEX idx_microservices_nats_rule_id ON Microservices (nats_rule_id); +CREATE INDEX idx_microservices_nats_account_id ON Microservices (nats_account_id); +CREATE INDEX idx_microservices_nats_user_id ON Microservices (nats_user_id); + +CREATE TABLE IF NOT EXISTS AuthUsers ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + password_history_hashes TEXT, + must_change_password BOOLEAN DEFAULT false, + is_bootstrap BOOLEAN DEFAULT false, + failed_attempts INT DEFAULT 0, + locked_until DATETIME, + deleted_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_users_email ON AuthUsers (email); +CREATE INDEX idx_auth_users_deleted_at ON AuthUsers (deleted_at); + +CREATE TABLE IF NOT EXISTS AuthGroups ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL UNIQUE, + is_system BOOLEAN DEFAULT false, + mfa_required BOOLEAN NOT NULL DEFAULT false, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthUserGroups ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id VARCHAR(36) NOT NULL, + group_id INT NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE, + FOREIGN KEY (group_id) REFERENCES AuthGroups (id) ON DELETE CASCADE, + UNIQUE KEY uk_auth_user_groups_user_group (user_id, group_id) +); + +CREATE INDEX idx_auth_user_groups_user_id ON AuthUserGroups (user_id); +CREATE INDEX idx_auth_user_groups_group_id ON AuthUserGroups (group_id); + +CREATE TABLE IF NOT EXISTS AuthMfa ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id VARCHAR(36) NOT NULL UNIQUE, + totp_secret_encrypted TEXT, + enabled BOOLEAN DEFAULT false, + recovery_codes_hash TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_mfa_user_id ON AuthMfa (user_id); + +CREATE TABLE IF NOT EXISTS AuthPasswordResetSessions ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_password_reset_sessions_user_id ON AuthPasswordResetSessions (user_id); +CREATE INDEX idx_auth_password_reset_sessions_expires_at ON AuthPasswordResetSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthRefreshTokens ( + id INT AUTO_INCREMENT PRIMARY KEY, + token_hash VARCHAR(255) NOT NULL, + user_id VARCHAR(36) NOT NULL, + family_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + revoked BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_refresh_tokens_token_hash ON AuthRefreshTokens (token_hash); +CREATE INDEX idx_auth_refresh_tokens_user_id ON AuthRefreshTokens (user_id); +CREATE INDEX idx_auth_refresh_tokens_family_id ON AuthRefreshTokens (family_id); +CREATE INDEX idx_auth_refresh_tokens_expires_at ON AuthRefreshTokens (expires_at); + +CREATE TABLE IF NOT EXISTS AuthOidcKeys ( + id INT AUTO_INCREMENT PRIMARY KEY, + kid VARCHAR(255) NOT NULL UNIQUE, + key_material_encrypted TEXT, + vault_ref TEXT, + active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_oidc_keys_active ON AuthOidcKeys (active); + +CREATE TABLE IF NOT EXISTS AuthOidcClients ( + id INT AUTO_INCREMENT PRIMARY KEY, + client_id VARCHAR(255) NOT NULL UNIQUE, + secret_ref TEXT, + client_type VARCHAR(32) NOT NULL DEFAULT 'confidential', + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthOidcProviderStates ( + id INT AUTO_INCREMENT PRIMARY KEY, + model VARCHAR(64) NOT NULL, + record_id VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME, + grant_id VARCHAR(255), + uid VARCHAR(255), + user_code VARCHAR(255), + consumed BOOLEAN DEFAULT false, + consumed_at DATETIME, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY uq_auth_oidc_provider_states_model_record (model, record_id) +); + +CREATE INDEX idx_auth_oidc_provider_states_grant_id ON AuthOidcProviderStates (grant_id); +CREATE INDEX idx_auth_oidc_provider_states_uid ON AuthOidcProviderStates (uid); +CREATE INDEX idx_auth_oidc_provider_states_user_code ON AuthOidcProviderStates (user_code); +CREATE INDEX idx_auth_oidc_provider_states_expires_at ON AuthOidcProviderStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBffSessions ( + sid VARCHAR(255) PRIMARY KEY NOT NULL, + data TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_bff_sessions_expires_at ON AuthBffSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthInteractionStates ( + uid VARCHAR(255) PRIMARY KEY NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_interaction_states_expires_at ON AuthInteractionStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBootstrapMeta ( + id INT AUTO_INCREMENT PRIMARY KEY, + completed_at DATETIME, + bootstrap_admin_user_id VARCHAR(36), + session_secret_ref TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (bootstrap_admin_user_id) REFERENCES AuthUsers (id) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS AuthPolicy ( + id INT PRIMARY KEY, + min_password_length INT DEFAULT 12, + require_uppercase BOOLEAN DEFAULT true, + require_lowercase BOOLEAN DEFAULT true, + require_digit BOOLEAN DEFAULT true, + password_max_age_days INT DEFAULT 0, + password_history_count INT DEFAULT 5, + max_failed_attempts INT DEFAULT 5, + lockout_duration_minutes INT DEFAULT 15, + access_token_ttl_seconds INT DEFAULT 900, + refresh_token_ttl_seconds INT DEFAULT 3600, + refresh_rotation BOOLEAN DEFAULT true, + max_concurrent_sessions INT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS FogPlatformSpecs ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + spec_json TEXT NOT NULL, + generation INT NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformStatuses ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + observed_generation INT NOT NULL DEFAULT 0, + phase VARCHAR(32) NOT NULL DEFAULT 'Pending', + last_error TEXT, + last_transition_at DATETIME, + conditions_json TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformReconcileTasks ( + id INT AUTO_INCREMENT PRIMARY KEY, + fog_uuid VARCHAR(36) NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_generation INT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_platform_reconcile_tasks_fog_status ON FogPlatformReconcileTasks (fog_uuid, status); +CREATE INDEX idx_fog_platform_reconcile_tasks_status_claimed ON FogPlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_fog_platform_reconcile_tasks_next_attempt ON FogPlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ServicePlatformReconcileTasks ( + id INT AUTO_INCREMENT PRIMARY KEY, + service_name TEXT NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_snapshot TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_service_platform_reconcile_tasks_name_status ON ServicePlatformReconcileTasks (service_name(255), status); +CREATE INDEX idx_service_platform_reconcile_tasks_status_claimed ON ServicePlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_service_platform_reconcile_tasks_next_attempt ON ServicePlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ReconcileOutbox ( + id INT AUTO_INCREMENT PRIMARY KEY, + kind VARCHAR(32) NOT NULL, + payload TEXT NOT NULL, + idempotency_key VARCHAR(255) NOT NULL, + created_at DATETIME, + processed_at DATETIME, + last_error TEXT, + UNIQUE KEY uk_reconcile_outbox_idempotency_key (idempotency_key) +); + +CREATE INDEX idx_reconcile_outbox_unprocessed ON ReconcileOutbox (processed_at, id); + +CREATE TABLE IF NOT EXISTS HubRouterConfigLocks ( + id INT PRIMARY KEY, + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_models_registry_id ON Models (registry_id); +CREATE INDEX idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(64) NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX idx_microservice_model_items_name ON MicroserviceModelItems (name); + +COMMIT; diff --git a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql new file mode 100644 index 00000000..fde82076 --- /dev/null +++ b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql @@ -0,0 +1,1345 @@ +CREATE TABLE IF NOT EXISTS "Applications" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255) DEFAULT '', + is_activated BOOLEAN DEFAULT false, + is_system BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + nats_access BOOLEAN DEFAULT false, + nats_rule_id INTEGER +); + +CREATE TABLE IF NOT EXISTS "Registries" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + url VARCHAR(255), + is_public BOOLEAN, + user_name TEXT, + password TEXT, + user_email TEXT, + type VARCHAR(16) NOT NULL DEFAULT 'oci', + ca TEXT, + insecure BOOLEAN NOT NULL DEFAULT false +); + +CREATE TABLE IF NOT EXISTS "CatalogItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255), + category TEXT, + config_example VARCHAR(255) DEFAULT '{}', + publisher TEXT, + disk_required BIGINT DEFAULT 0, + ram_required BIGINT DEFAULT 0, + picture VARCHAR(255) DEFAULT 'images/shared/default.png', + is_public BOOLEAN DEFAULT false, + registry_id INT, + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX idx_catalog_item_registry_id ON "CatalogItems" (registry_id); + +CREATE TABLE IF NOT EXISTS "Architectures" ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_architecture_network_catalog_item_id ON "Architectures" (network_catalog_item_id); + +CREATE TABLE IF NOT EXISTS "Fogs" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) DEFAULT 'Unnamed ioFog 1', + location TEXT, + gps_mode TEXT, + latitude DOUBLE PRECISION, + longitude DOUBLE PRECISION, + description TEXT, + last_active BIGINT, + daemon_status VARCHAR(36) DEFAULT 'NOT_PROVISIONED', + daemon_operating_duration BIGINT DEFAULT 0, + daemon_last_start BIGINT, + memory_usage DOUBLE PRECISION DEFAULT 0.000, + disk_usage DOUBLE PRECISION DEFAULT 0.000, + cpu_usage DOUBLE PRECISION DEFAULT 0.00, + memory_violation TEXT, + disk_violation TEXT, + cpu_violation TEXT, + system_available_disk BIGINT, + system_available_memory BIGINT, + system_total_cpu DOUBLE PRECISION, + security_status VARCHAR(36) DEFAULT 'OK', + security_violation_info VARCHAR(36) DEFAULT 'No violation', + catalog_item_status TEXT, + repository_count BIGINT DEFAULT 0, + repository_status TEXT, + system_time BIGINT, + last_status_time BIGINT, + ip_address VARCHAR(36) DEFAULT '0.0.0.0', + ip_address_external VARCHAR(36) DEFAULT '0.0.0.0', + host VARCHAR(36), + catalog_item_message_counts TEXT, + last_command_time BIGINT, + network_interface VARCHAR(36) DEFAULT 'dynamic', + docker_url VARCHAR(255) DEFAULT 'unix:///run/edgelet/contaienrd.sock', + disk_limit DOUBLE PRECISION DEFAULT 50, + disk_directory VARCHAR(255) DEFAULT '/var/lib/iofog-agent/', + memory_limit DOUBLE PRECISION DEFAULT 4096, + cpu_limit DOUBLE PRECISION DEFAULT 80, + log_limit DOUBLE PRECISION DEFAULT 10, + log_directory VARCHAR(255) DEFAULT '/var/log/iofog/', + log_file_count BIGINT DEFAULT 10, + version TEXT, + is_ready_to_upgrade BOOLEAN DEFAULT TRUE, + is_ready_to_rollback BOOLEAN DEFAULT FALSE, + status_frequency INT DEFAULT 10, + change_frequency INT DEFAULT 20, + tunnel VARCHAR(255) DEFAULT '', + isolated_docker_container BOOLEAN DEFAULT FALSE, + docker_pruning_freq INT DEFAULT 0, + available_disk_threshold DOUBLE PRECISION DEFAULT 20, + log_level VARCHAR(10) DEFAULT 'INFO', + is_system BOOLEAN DEFAULT FALSE, + router_id INT DEFAULT 0, + time_zone VARCHAR(36) DEFAULT 'Etc/UTC', + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + arch_id INT DEFAULT 0, + container_engine VARCHAR(36), + deployment_type VARCHAR(36), + active_volume_mounts BIGINT DEFAULT 0, + volume_mount_last_update BIGINT DEFAULT 0, + warning_message TEXT, + gps_device VARCHAR(36), + gps_scan_frequency INT DEFAULT 60, + edge_guard_frequency INT DEFAULT 0, + gps_status VARCHAR(32), + nats_id INT, + available_runtimes TEXT, + model_status TEXT, + runtime_classes TEXT, + available_cdi_devices TEXT, + active_models BIGINT DEFAULT 0, + model_last_update BIGINT DEFAULT 0, + FOREIGN KEY (arch_id) REFERENCES "Architectures" (id) +); + +CREATE INDEX idx_fog_arch_id ON "Fogs" (arch_id); + +CREATE TABLE IF NOT EXISTS "ChangeTrackings" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_config BOOLEAN DEFAULT false, + reboot BOOLEAN DEFAULT false, + deletenode BOOLEAN DEFAULT false, + version BOOLEAN DEFAULT false, + microservice_list BOOLEAN DEFAULT false, + config BOOLEAN DEFAULT false, + registries BOOLEAN DEFAULT false, + tunnel BOOLEAN DEFAULT false, + router_changed BOOLEAN DEFAULT false, + prune BOOLEAN DEFAULT false, + last_updated VARCHAR(255) DEFAULT false, + iofog_uuid VARCHAR(36), + volume_mounts BOOLEAN DEFAULT false, + exec_sessions BOOLEAN DEFAULT false, + microservice_logs BOOLEAN DEFAULT false, + fog_logs BOOLEAN DEFAULT false, + models BOOLEAN DEFAULT false, + runtime_classes BOOLEAN DEFAULT false, + microservice_models BOOLEAN DEFAULT false, + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_change_tracking_iofog_uuid ON "ChangeTrackings" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "FogProvisionKeys" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + provisioning_string VARCHAR(100), + expiration_time BIGINT, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_provision_keys_iofogUuid ON "FogProvisionKeys" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "FogVersionCommands" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + version_command VARCHAR(100), + semver VARCHAR(100), + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_version_commands_iofogUuid ON "FogVersionCommands" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "Tunnels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + username TEXT, + password TEXT, + host TEXT, + remote_port INT, + local_port INT DEFAULT 22, + rsa_key TEXT, + closed BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_tunnels_iofogUuid ON "Tunnels" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "Microservices" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + config TEXT, + name VARCHAR(255) DEFAULT 'New Microservice', + config_last_updated BIGINT, + rebuild BOOLEAN DEFAULT false, + log_size BIGINT DEFAULT 0, + delete BOOLEAN DEFAULT false, + delete_with_cleanup BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + catalog_item_id INT, + registry_id INT DEFAULT 1, + iofog_uuid VARCHAR(36), + application_id INT, + run_as_user TEXT, + platform TEXT, + runtime TEXT, + annotations TEXT, + pid_mode VARCHAR(36), + ipc_mode VARCHAR(36), + schedule INT DEFAULT 50, + cpu_set_cpus TEXT, + memory_limit DOUBLE PRECISION, + is_activated BOOLEAN DEFAULT true, + host_network_mode BOOLEAN DEFAULT false, + is_privileged BOOLEAN DEFAULT false, + nats_rule_id INT, + nats_access BOOLEAN DEFAULT false, + nats_account_id INT, + nats_user_id INT, + nats_creds_secret_name TEXT, + is_controller BOOLEAN DEFAULT false, + run_as_group TEXT, + read_only_root_filesystem BOOLEAN DEFAULT false, + working_dir TEXT, + cpus DOUBLE PRECISION, + memory_reservation DOUBLE PRECISION, + memory_swap DOUBLE PRECISION, + shm_size DOUBLE PRECISION, + sysctls TEXT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE, + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL, + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES "Applications" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservices_catalogItemId ON "Microservices" (catalog_item_id); +CREATE INDEX idx_microservices_registryId ON "Microservices" (registry_id); +CREATE INDEX idx_microservices_iofogUuid ON "Microservices" (iofog_uuid); +CREATE INDEX idx_microservices_applicationId ON "Microservices" (application_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceArgs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cmd TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_args_microserviceUuid ON "MicroserviceArgs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceEnvs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + key TEXT, + value TEXT, + microservice_uuid VARCHAR(36), + value_from_secret TEXT, + value_from_config_map TEXT, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_envs_microserviceUuid ON "MicroserviceEnvs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceExtraHost" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + template_type TEXT, + name TEXT, + template TEXT, + value TEXT, + microservice_uuid VARCHAR(36), + target_microservice_uuid VARCHAR(36), + target_fog_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_extra_host_microserviceUuid ON "MicroserviceExtraHost" (microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetMicroserviceUuid ON "MicroserviceExtraHost" (target_microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetFogUuid ON "MicroserviceExtraHost" (target_fog_uuid); + +CREATE TABLE IF NOT EXISTS "MicroservicePorts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + port_internal INT, + port_external INT, + is_udp BOOLEAN, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_port_microserviceUuid ON "MicroservicePorts" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'QUEUED', + operating_duration BIGINT DEFAULT 0, + start_time BIGINT DEFAULT 0, + cpu_usage DOUBLE PRECISION DEFAULT 0.000, + memory_usage BIGINT DEFAULT 0, + container_id VARCHAR(255) DEFAULT '', + percentage DOUBLE PRECISION DEFAULT 0.00, + error_message TEXT, + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + ip_address TEXT, + exec_session_ids TEXT, + health_status TEXT, + pod_id TEXT DEFAULT '', + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_status_microserviceUuid ON "MicroserviceStatuses" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "VolumeMappings" ( + uuid INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + host_destination TEXT, + container_destination TEXT, + access_mode TEXT, + type TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mappings_microserviceUuid ON "VolumeMappings" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "CatalogItemImages" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + container_image TEXT, + catalog_item_id INT, + microservice_uuid VARCHAR(36), + arch_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (arch_id) REFERENCES "Architectures" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_image_catalog_item_id ON "CatalogItemImages" (catalog_item_id); +CREATE INDEX idx_catalog_item_image_microservice_uuid ON "CatalogItemImages" (microservice_uuid); +CREATE INDEX idx_catalog_item_image_arch_id ON "CatalogItemImages" (arch_id); + +CREATE TABLE IF NOT EXISTS "CatalogItemInputTypes" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_input_type_catalog_item_id ON "CatalogItemInputTypes" (catalog_item_id); + +CREATE TABLE IF NOT EXISTS "CatalogItemOutputTypes" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_output_type_catalog_item_id ON "CatalogItemOutputTypes" (catalog_item_id); + +CREATE TABLE IF NOT EXISTS "Routers" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + is_edge BOOLEAN DEFAULT true, + messaging_port INT DEFAULT 5671, + edge_router_port INT, + inter_router_port INT, + host TEXT, + is_default BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_router_iofogUuid ON "Routers" (iofog_uuid); + +CREATE TABLE "RouterConnections" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_router INT, + dest_router INT, + created_at TIMESTAMPTZ(0) NOT NULL, + updated_at TIMESTAMPTZ(0) NOT NULL, + FOREIGN KEY (source_router) REFERENCES "Routers"(id) ON DELETE CASCADE, + FOREIGN KEY (dest_router) REFERENCES "Routers"(id) ON DELETE CASCADE +); + +CREATE INDEX idx_routerconnections_sourceRouter ON "RouterConnections" (source_router); +CREATE INDEX idx_routerconnections_destRouter ON "RouterConnections" (dest_router); + +CREATE TABLE IF NOT EXISTS "Config" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + key VARCHAR(255) NOT NULL UNIQUE, + value VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_config_key ON "Config" (key); + +CREATE TABLE IF NOT EXISTS "Tags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + value VARCHAR(255) UNIQUE NOT NULL +); + +CREATE TABLE IF NOT EXISTS "IofogTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_iofogtags_fog_uuid ON "IofogTags" (fog_uuid); +CREATE INDEX idx_iofogtags_tag_id ON "IofogTags" (tag_id); + +CREATE TABLE IF NOT EXISTS "ApplicationTemplates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL DEFAULT 'new-application', + description VARCHAR(255) DEFAULT '', + schema_version VARCHAR(255) DEFAULT '', + application_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "ApplicationTemplateVariables" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + application_template_id INT NOT NULL, + key TEXT, + description VARCHAR(255) DEFAULT '', + default_value VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (application_template_id) REFERENCES "ApplicationTemplates" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_applicationtemplatevariables_application_template_id ON "ApplicationTemplateVariables" (application_template_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceCdiDevices" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cdi_devices TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_cdiDevices_microserviceUuid ON "MicroserviceCdiDevices" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroservicePubTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceSubTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicepubtags_microservice_uuid ON "MicroservicePubTags" (microservice_uuid); +CREATE INDEX idx_microservicesubtags_microservice_uuid ON "MicroserviceSubTags" (microservice_uuid); +CREATE INDEX idx_microservicepubtags_tag_id ON "MicroservicePubTags" (tag_id); +CREATE INDEX idx_microservicesubtags_tag_id ON "MicroserviceSubTags" (tag_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceCapAdd" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cap_add TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capAdd_microserviceUuid ON "MicroserviceCapAdd" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceCapDrop" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cap_drop TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capDrop_microserviceUuid ON "MicroserviceCapDrop" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "FogPublicKeys" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + public_key TEXT, + iofog_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_public_keys_iofogUuid ON "FogPublicKeys" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "FogUsedTokens" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + jti VARCHAR(255) NOT NULL, + iofog_uuid VARCHAR(36), + expiry_time BIGINT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_used_tokens_iofogUuid ON "FogUsedTokens" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "Secrets" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('Opaque', 'tls')), + data TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_secrets_name ON "Secrets" (name); + +CREATE TABLE IF NOT EXISTS "Certificates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + subject TEXT NOT NULL, + is_ca BOOLEAN DEFAULT false, + signed_by_id INT, + hosts TEXT, + valid_from TIMESTAMPTZ(0) NOT NULL, + valid_to TIMESTAMPTZ(0) NOT NULL, + serial_number TEXT NOT NULL, + secret_id INT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (signed_by_id) REFERENCES "Certificates" (id) ON DELETE SET NULL, + FOREIGN KEY (secret_id) REFERENCES "Secrets" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_certificates_name ON "Certificates" (name); +CREATE INDEX idx_certificates_valid_to ON "Certificates" (valid_to); +CREATE INDEX idx_certificates_is_ca ON "Certificates" (is_ca); +CREATE INDEX idx_certificates_signed_by_id ON "Certificates" (signed_by_id); +CREATE INDEX idx_certificates_secret_id ON "Certificates" (secret_id); + +CREATE TABLE IF NOT EXISTS "Services" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + type TEXT NOT NULL, + resource TEXT NOT NULL, + target_port INTEGER NOT NULL, + service_port INTEGER, + k8s_type TEXT, + bridge_port INTEGER, + default_bridge TEXT, + service_endpoint TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + provisioning_status VARCHAR(36) DEFAULT 'pending', + provisioning_error TEXT +); + +CREATE INDEX idx_services_name ON "Services" (name); +CREATE INDEX idx_services_id ON "Services" (id); + +CREATE TABLE IF NOT EXISTS "ServiceTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + service_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (service_id) REFERENCES "Services" (id) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_service_tags_service_id ON "ServiceTags" (service_id); +CREATE INDEX idx_service_tags_tag_id ON "ServiceTags" (tag_id); + +CREATE TABLE IF NOT EXISTS "ConfigMaps" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + immutable BOOLEAN DEFAULT false, + data TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + use_vault BOOLEAN DEFAULT true +); + +CREATE INDEX idx_config_maps_name ON "ConfigMaps" (name); + +CREATE TABLE IF NOT EXISTS "VolumeMounts" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + config_map_name VARCHAR(255), + secret_name VARCHAR(255), + version INT DEFAULT 1, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (config_map_name) REFERENCES "ConfigMaps" (name) ON DELETE CASCADE, + FOREIGN KEY (secret_name) REFERENCES "Secrets" (name) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mounts_uuid ON "VolumeMounts" (uuid); +CREATE INDEX idx_volume_mounts_config_map_name ON "VolumeMounts" (config_map_name); +CREATE INDEX idx_volume_mounts_secret_name ON "VolumeMounts" (secret_name); + +CREATE TABLE IF NOT EXISTS "FogVolumeMounts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + volume_mount_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (volume_mount_uuid) REFERENCES "VolumeMounts" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_volume_mounts_fog_uuid ON "FogVolumeMounts" (fog_uuid); +CREATE INDEX idx_fog_volume_mounts_volume_mount_uuid ON "FogVolumeMounts" (volume_mount_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceExecStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'INACTIVE', + exec_session_id VARCHAR(255), + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_status_microservice_uuid ON "MicroserviceExecStatuses" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceHealthChecks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + test TEXT, + interval DOUBLE PRECISION, + timeout DOUBLE PRECISION, + start_period DOUBLE PRECISION, + start_interval DOUBLE PRECISION, + retries INT, + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_health_check_microservice_uuid ON "MicroserviceHealthChecks" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "Events" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + timestamp BIGINT NOT NULL, + event_type VARCHAR(20) NOT NULL, + endpoint_type VARCHAR(10) NOT NULL, + actor_id VARCHAR(255), + method VARCHAR(10), + resource_type VARCHAR(50), + resource_id VARCHAR(255), + endpoint_path TEXT NOT NULL, + ip_address VARCHAR(45), + status VARCHAR(20) NOT NULL, + status_code INT, + status_message TEXT, + request_id VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_events_timestamp ON "Events" (timestamp); +CREATE INDEX idx_events_endpoint_type ON "Events" (endpoint_type); +CREATE INDEX idx_events_actor_id ON "Events" (actor_id); +CREATE INDEX idx_events_resource_type ON "Events" (resource_type); +CREATE INDEX idx_events_status ON "Events" (status); +CREATE INDEX idx_events_method ON "Events" (method); +CREATE INDEX idx_events_event_type ON "Events" (event_type); +CREATE INDEX idx_events_created_at ON "Events" (created_at); + +CREATE TABLE IF NOT EXISTS "MicroserviceLogStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_log_status_microservice_uuid ON "MicroserviceLogStatuses" (microservice_uuid); +CREATE INDEX idx_microservice_log_status_session_id ON "MicroserviceLogStatuses" (session_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceExecSessions" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON "MicroserviceExecSessions" (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON "MicroserviceExecSessions" (session_id); + +CREATE TABLE IF NOT EXISTS "FogLogStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + iofog_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_log_status_iofog_uuid ON "FogLogStatuses" (iofog_uuid); +CREATE INDEX idx_fog_log_status_session_id ON "FogLogStatuses" (session_id); + +CREATE TABLE IF NOT EXISTS "RbacRoles" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT UNIQUE NOT NULL, + kind TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "RbacRoleRules" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + role_id INT NOT NULL, + api_groups TEXT NOT NULL, + resources TEXT NOT NULL, + verbs TEXT NOT NULL, + resource_names TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (role_id) REFERENCES "RbacRoles" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "RbacRoleBindings" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT UNIQUE NOT NULL, + kind TEXT, + role_ref TEXT NOT NULL, + subjects TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + role_id INTEGER +); + +CREATE TABLE IF NOT EXISTS "RbacServiceAccounts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL, + role_ref TEXT, + role_id INT REFERENCES "RbacRoles" (id), + microservice_uuid VARCHAR(36) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + application_id INT REFERENCES "Applications" (id) ON DELETE SET NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_rbac_role_rules_role_id ON "RbacRoleRules" (role_id); +CREATE INDEX idx_rbac_roles_name ON "RbacRoles" (name); +CREATE INDEX idx_rbac_role_bindings_name ON "RbacRoleBindings" (name); +CREATE INDEX idx_rbac_service_accounts_name ON "RbacServiceAccounts" (name); + +CREATE TABLE IF NOT EXISTS "RbacCacheVersion" ( + id INT PRIMARY KEY DEFAULT 1, + version BIGINT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + CONSTRAINT single_row CHECK (id = 1) +); + +CREATE INDEX idx_rbac_role_bindings_role_id ON "RbacRoleBindings" (role_id); + +CREATE INDEX idx_rbac_service_accounts_role_id ON "RbacServiceAccounts" (role_id); +CREATE UNIQUE INDEX idx_rbac_service_accounts_microservice_uuid_unique ON "RbacServiceAccounts" (microservice_uuid) WHERE microservice_uuid IS NOT NULL; +CREATE UNIQUE INDEX idx_rbac_service_accounts_application_id_name_unique ON "RbacServiceAccounts" (application_id, name); + +CREATE TABLE IF NOT EXISTS "ClusterControllers" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + host VARCHAR(255), + process_id INT, + last_heartbeat TIMESTAMPTZ(0), + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_cluster_controllers_uuid ON "ClusterControllers" (uuid); +CREATE INDEX idx_cluster_controllers_host ON "ClusterControllers" (host); +CREATE INDEX idx_cluster_controllers_active ON "ClusterControllers" (is_active, last_heartbeat); + +CREATE TABLE IF NOT EXISTS "NatsOperators" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "NatsAccounts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + is_system BOOLEAN DEFAULT false, + is_leaf_system BOOLEAN DEFAULT false, + operator_id INT NOT NULL, + application_id INT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (operator_id) REFERENCES "NatsOperators" (id) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES "Applications" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "NatsUsers" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + creds_secret_name TEXT NOT NULL, + is_bearer BOOLEAN DEFAULT false, + account_id INT NOT NULL, + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + nats_user_rule_id INTEGER, + FOREIGN KEY (account_id) REFERENCES "NatsAccounts" (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS "NatsInstances" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + iofog_uuid VARCHAR(36), + is_leaf BOOLEAN DEFAULT true, + is_hub BOOLEAN DEFAULT false, + host TEXT, + server_port INT, + leaf_port INT, + cluster_port INT, + mqtt_port INT, + http_port INT, + configmap_name TEXT, + jwt_dir_mount_name TEXT, + cert_secret_name TEXT, + js_storage_size TEXT, + js_memory_store_size TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "NatsConnections" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + source_nats INT NOT NULL, + dest_nats INT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (source_nats) REFERENCES "NatsInstances" (id) ON DELETE CASCADE, + FOREIGN KEY (dest_nats) REFERENCES "NatsInstances" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "NatsReconcileTasks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + reason VARCHAR(64) NOT NULL, + application_id INT, + account_rule_id INT, + user_rule_id INT, + fog_uuids TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "NatsAccountRules" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + info_url TEXT, + max_connections INT, + max_leaf_node_connections INT, + max_data BIGINT, + max_exports INT, + max_imports INT, + max_msg_payload INT, + max_subscriptions INT, + exports_allow_wildcards BOOLEAN DEFAULT true, + disallow_bearer BOOLEAN, + response_permissions TEXT, + resp_max INT, + resp_ttl BIGINT, + imports TEXT, + exports TEXT, + mem_storage BIGINT, + disk_storage BIGINT, + streams INT, + consumer INT, + max_ack_pending INT, + mem_max_stream_bytes BIGINT, + disk_max_stream_bytes BIGINT, + max_bytes_required BOOLEAN, + tiered_limits TEXT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "NatsUserRules" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + max_subscriptions INT, + max_payload INT, + max_data BIGINT, + bearer_token BOOLEAN DEFAULT false, + proxy_required BOOLEAN, + allowed_connection_types TEXT, + src TEXT, + times TEXT, + times_location TEXT, + resp_max INT, + resp_ttl BIGINT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + tags TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE UNIQUE INDEX idx_nats_accounts_application_id_unique ON "NatsAccounts" (application_id) WHERE application_id IS NOT NULL; +CREATE INDEX idx_nats_accounts_application_id ON "NatsAccounts" (application_id); +CREATE UNIQUE INDEX idx_nats_users_account_id_name ON "NatsUsers" (account_id, name); +CREATE INDEX idx_nats_users_account_id ON "NatsUsers" (account_id); +CREATE INDEX idx_nats_users_microservice_uuid ON "NatsUsers" (microservice_uuid); +CREATE INDEX idx_nats_users_nats_user_rule_id ON "NatsUsers" (nats_user_rule_id); +CREATE UNIQUE INDEX idx_nats_instances_iofog_uuid_unique ON "NatsInstances" (iofog_uuid); +CREATE INDEX idx_nats_instances_iofog_uuid ON "NatsInstances" (iofog_uuid); +CREATE UNIQUE INDEX idx_nats_connections_source_dest_unique ON "NatsConnections" (source_nats, dest_nats); +CREATE INDEX idx_nats_connections_source_nats ON "NatsConnections" (source_nats); +CREATE INDEX idx_nats_connections_dest_nats ON "NatsConnections" (dest_nats); +CREATE INDEX idx_nats_account_rules_name ON "NatsAccountRules" (name); +CREATE INDEX idx_nats_user_rules_name ON "NatsUserRules" (name); +CREATE INDEX idx_nats_reconcile_tasks_status_claimed ON "NatsReconcileTasks" (status, claimed_at); + +CREATE INDEX idx_applications_nats_rule_id ON "Applications" (nats_rule_id); +CREATE INDEX idx_microservices_nats_rule_id ON "Microservices" (nats_rule_id); +CREATE INDEX idx_microservices_nats_account_id ON "Microservices" (nats_account_id); +CREATE INDEX idx_microservices_nats_user_id ON "Microservices" (nats_user_id); + +CREATE TABLE IF NOT EXISTS "AuthUsers" ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + password_history_hashes TEXT, + must_change_password BOOLEAN DEFAULT false, + is_bootstrap BOOLEAN DEFAULT false, + failed_attempts INT DEFAULT 0, + locked_until TIMESTAMPTZ(0), + deleted_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_auth_users_email ON "AuthUsers" (email); +CREATE INDEX idx_auth_users_deleted_at ON "AuthUsers" (deleted_at); + +CREATE TABLE IF NOT EXISTS "AuthGroups" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + is_system BOOLEAN DEFAULT false, + mfa_required BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "AuthUserGroups" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + group_id INT NOT NULL, + created_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE, + FOREIGN KEY (group_id) REFERENCES "AuthGroups" (id) ON DELETE CASCADE, + UNIQUE (user_id, group_id) +); + +CREATE INDEX idx_auth_user_groups_user_id ON "AuthUserGroups" (user_id); +CREATE INDEX idx_auth_user_groups_group_id ON "AuthUserGroups" (group_id); + +CREATE TABLE IF NOT EXISTS "AuthMfa" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL UNIQUE, + totp_secret_encrypted TEXT, + enabled BOOLEAN DEFAULT false, + recovery_codes_hash TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_mfa_user_id ON "AuthMfa" (user_id); + +CREATE TABLE IF NOT EXISTS "AuthPasswordResetSessions" ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + expires_at TIMESTAMPTZ(0) NOT NULL, + created_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_password_reset_sessions_user_id ON "AuthPasswordResetSessions" (user_id); +CREATE INDEX idx_auth_password_reset_sessions_expires_at ON "AuthPasswordResetSessions" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthRefreshTokens" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + token_hash VARCHAR(255) NOT NULL, + user_id VARCHAR(36) NOT NULL, + family_id VARCHAR(36) NOT NULL, + expires_at TIMESTAMPTZ(0) NOT NULL, + revoked BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_refresh_tokens_token_hash ON "AuthRefreshTokens" (token_hash); +CREATE INDEX idx_auth_refresh_tokens_user_id ON "AuthRefreshTokens" (user_id); +CREATE INDEX idx_auth_refresh_tokens_family_id ON "AuthRefreshTokens" (family_id); +CREATE INDEX idx_auth_refresh_tokens_expires_at ON "AuthRefreshTokens" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthOidcKeys" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + kid VARCHAR(255) NOT NULL UNIQUE, + key_material_encrypted TEXT, + vault_ref TEXT, + active BOOLEAN DEFAULT true, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_auth_oidc_keys_active ON "AuthOidcKeys" (active); + +CREATE TABLE IF NOT EXISTS "AuthOidcClients" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + client_id VARCHAR(255) NOT NULL UNIQUE, + secret_ref TEXT, + client_type VARCHAR(32) NOT NULL DEFAULT 'confidential', + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "AuthOidcProviderStates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + model VARCHAR(64) NOT NULL, + record_id VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + expires_at TIMESTAMPTZ(0), + grant_id VARCHAR(255), + uid VARCHAR(255), + user_code VARCHAR(255), + consumed BOOLEAN DEFAULT false, + consumed_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + UNIQUE (model, record_id) +); + +CREATE INDEX idx_auth_oidc_provider_states_grant_id ON "AuthOidcProviderStates" (grant_id); +CREATE INDEX idx_auth_oidc_provider_states_uid ON "AuthOidcProviderStates" (uid); +CREATE INDEX idx_auth_oidc_provider_states_user_code ON "AuthOidcProviderStates" (user_code); +CREATE INDEX idx_auth_oidc_provider_states_expires_at ON "AuthOidcProviderStates" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthBffSessions" ( + sid VARCHAR(255) PRIMARY KEY NOT NULL, + data TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ +); + +CREATE INDEX idx_auth_bff_sessions_expires_at ON "AuthBffSessions" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthInteractionStates" ( + uid VARCHAR(255) PRIMARY KEY NOT NULL, + payload TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ +); + +CREATE INDEX idx_auth_interaction_states_expires_at ON "AuthInteractionStates" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthBootstrapMeta" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + completed_at TIMESTAMPTZ(0), + bootstrap_admin_user_id VARCHAR(36), + session_secret_ref TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (bootstrap_admin_user_id) REFERENCES "AuthUsers" (id) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS "AuthPolicy" ( + id INT PRIMARY KEY NOT NULL CHECK (id = 1), + min_password_length INT DEFAULT 12, + require_uppercase BOOLEAN DEFAULT true, + require_lowercase BOOLEAN DEFAULT true, + require_digit BOOLEAN DEFAULT true, + password_max_age_days INT DEFAULT 0, + password_history_count INT DEFAULT 5, + max_failed_attempts INT DEFAULT 5, + lockout_duration_minutes INT DEFAULT 15, + access_token_ttl_seconds INT DEFAULT 900, + refresh_token_ttl_seconds INT DEFAULT 3600, + refresh_rotation BOOLEAN DEFAULT true, + max_concurrent_sessions INT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "FogPlatformSpecs" ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + spec_json TEXT NOT NULL, + generation INT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "FogPlatformStatuses" ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + observed_generation INT NOT NULL DEFAULT 0, + phase VARCHAR(32) NOT NULL DEFAULT 'Pending', + last_error TEXT, + last_transition_at TIMESTAMPTZ(0), + conditions_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "FogPlatformReconcileTasks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36) NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_generation INT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + next_attempt_at TIMESTAMPTZ(0), + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX idx_fog_platform_reconcile_tasks_active_fog_uuid ON "FogPlatformReconcileTasks" (fog_uuid) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_fog_platform_reconcile_tasks_status_claimed ON "FogPlatformReconcileTasks" (status, claimed_at); +CREATE INDEX idx_fog_platform_reconcile_tasks_next_attempt ON "FogPlatformReconcileTasks" (next_attempt_at); + +CREATE TABLE IF NOT EXISTS "ServicePlatformReconcileTasks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + service_name TEXT NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_snapshot TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + next_attempt_at TIMESTAMPTZ(0), + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE UNIQUE INDEX idx_service_platform_reconcile_tasks_active_service_name ON "ServicePlatformReconcileTasks" (service_name) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_service_platform_reconcile_tasks_status_claimed ON "ServicePlatformReconcileTasks" (status, claimed_at); +CREATE INDEX idx_service_platform_reconcile_tasks_next_attempt ON "ServicePlatformReconcileTasks" (next_attempt_at); + +CREATE TABLE IF NOT EXISTS "ReconcileOutbox" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + kind VARCHAR(32) NOT NULL, + payload TEXT NOT NULL, + idempotency_key VARCHAR(255) NOT NULL UNIQUE, + created_at TIMESTAMPTZ(0), + processed_at TIMESTAMPTZ(0), + last_error TEXT +); + +CREATE INDEX idx_reconcile_outbox_unprocessed ON "ReconcileOutbox" (processed_at, id); + +CREATE TABLE IF NOT EXISTS "HubRouterConfigLocks" ( + id INT PRIMARY KEY NOT NULL CHECK (id = 1), + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON "Registries" (url, type); + +CREATE TABLE IF NOT EXISTS "Models" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON "Models" (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON "Models" (name); + +CREATE TABLE IF NOT EXISTS "RuntimeClasses" ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS "FogModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES "Models" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON "FogModels" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON "FogModels" (model_uuid); + +CREATE TABLE IF NOT EXISTS "FogRuntimeClasses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES "RuntimeClasses" (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON "FogRuntimeClasses" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON "FogRuntimeClasses" (runtime_class_name); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplateVariables" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + key TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_template_id) REFERENCES "MicroserviceTemplates" (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON "MicroserviceTemplateVariables" (microservice_template_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceEntrypoints" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON "MicroserviceEntrypoints" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceDevices" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON "MicroserviceDevices" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceTmpfs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + container_path TEXT, + size DOUBLE PRECISION, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON "MicroserviceTmpfs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceUlimits" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + soft DOUBLE PRECISION, + hard DOUBLE PRECISION, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON "MicroserviceUlimits" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceModelItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Models" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON "MicroserviceModelItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON "MicroserviceModelItems" (name); diff --git a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql new file mode 100644 index 00000000..6c741e22 --- /dev/null +++ b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql @@ -0,0 +1,1344 @@ +CREATE TABLE IF NOT EXISTS Applications ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE, + description VARCHAR(255) DEFAULT '', + is_activated BOOLEAN DEFAULT false, + is_system BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + nats_access BOOLEAN DEFAULT false, + nats_rule_id INTEGER +); + +CREATE TABLE IF NOT EXISTS Registries ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + url VARCHAR(255), + is_public BOOLEAN, + user_name TEXT, + password TEXT, + user_email TEXT, + type VARCHAR(16) DEFAULT 'oci', + ca TEXT, + insecure BOOLEAN DEFAULT false +); + +CREATE TABLE IF NOT EXISTS CatalogItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE, + description VARCHAR(255), + category TEXT, + config_example VARCHAR(255) DEFAULT '{}', + publisher TEXT, + disk_required BIGINT DEFAULT 0, + ram_required BIGINT DEFAULT 0, + picture VARCHAR(255) DEFAULT 'images/shared/default.png', + is_public BOOLEAN DEFAULT false, + registry_id INT, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_catalog_item_registry_id ON CatalogItems (registry_id); + +CREATE TABLE IF NOT EXISTS Architectures ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_architecture_network_catalog_item_id ON Architectures (network_catalog_item_id); + +CREATE TABLE IF NOT EXISTS Fogs ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) DEFAULT 'Unnamed ioFog 1', + location TEXT, + gps_mode TEXT, + latitude FLOAT, + longitude FLOAT, + description TEXT, + last_active BIGINT, + daemon_status VARCHAR(36) DEFAULT 'NOT_PROVISIONED', + daemon_operating_duration BIGINT DEFAULT 0, + daemon_last_start BIGINT, + memory_usage FLOAT DEFAULT 0.000, + disk_usage FLOAT DEFAULT 0.000, + cpu_usage FLOAT DEFAULT 0.00, + memory_violation TEXT, + disk_violation TEXT, + cpu_violation TEXT, + system_available_disk BIGINT, + system_available_memory BIGINT, + system_total_cpu FLOAT, + security_status VARCHAR(36) DEFAULT 'OK', + security_violation_info VARCHAR(36) DEFAULT 'No violation', + catalog_item_status TEXT, + repository_count BIGINT DEFAULT 0, + repository_status TEXT, + system_time BIGINT, + last_status_time BIGINT, + ip_address VARCHAR(36) DEFAULT '0.0.0.0', + ip_address_external VARCHAR(36) DEFAULT '0.0.0.0', + host VARCHAR(36), + catalog_item_message_counts TEXT, + last_command_time BIGINT, + network_interface VARCHAR(36) DEFAULT 'dynamic', + docker_url VARCHAR(255) DEFAULT 'unix:///run/edgelet/contaienrd.sock', + disk_limit FLOAT DEFAULT 50, + disk_directory VARCHAR(255) DEFAULT '/var/lib/iofog-agent/', + memory_limit FLOAT DEFAULT 4096, + cpu_limit FLOAT DEFAULT 80, + log_limit FLOAT DEFAULT 10, + log_directory VARCHAR(255) DEFAULT '/var/log/iofog-agent/', + log_file_count BIGINT DEFAULT 10, + `version` TEXT, + is_ready_to_upgrade BOOLEAN DEFAULT TRUE, + is_ready_to_rollback BOOLEAN DEFAULT FALSE, + status_frequency INT DEFAULT 10, + change_frequency INT DEFAULT 20, + tunnel VARCHAR(255) DEFAULT '', + isolated_docker_container BOOLEAN DEFAULT FALSE, + docker_pruning_freq INT DEFAULT 0, + available_disk_threshold FLOAT DEFAULT 20, + log_level VARCHAR(10) DEFAULT 'INFO', + is_system BOOLEAN DEFAULT FALSE, + router_id INT DEFAULT 0, + time_zone VARCHAR(36) DEFAULT 'Etc/UTC', + created_at DATETIME, + updated_at DATETIME, + arch_id INT DEFAULT 0, + container_engine VARCHAR(36), + deployment_type VARCHAR(36), + active_volume_mounts BIGINT DEFAULT 0, + volume_mount_last_update BIGINT DEFAULT 0, + warning_message TEXT, + gps_device VARCHAR(36), + gps_scan_frequency INT DEFAULT 60, + edge_guard_frequency INT DEFAULT 0, + gps_status VARCHAR(32), + nats_id INT, + available_runtimes TEXT, + model_status TEXT, + runtime_classes TEXT, + available_cdi_devices TEXT, + active_models BIGINT DEFAULT 0, + model_last_update BIGINT DEFAULT 0, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) +); + +CREATE INDEX idx_fog_arch_id ON Fogs (arch_id); + +CREATE TABLE IF NOT EXISTS ChangeTrackings ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_config BOOLEAN DEFAULT false, + reboot BOOLEAN DEFAULT false, + deletenode BOOLEAN DEFAULT false, + version BOOLEAN DEFAULT false, + microservice_list BOOLEAN DEFAULT false, + config BOOLEAN DEFAULT false, + registries BOOLEAN DEFAULT false, + tunnel BOOLEAN DEFAULT false, + router_changed BOOLEAN DEFAULT false, + prune BOOLEAN DEFAULT false, + last_updated VARCHAR(255) DEFAULT false, + iofog_uuid VARCHAR(36), + volume_mounts BOOLEAN DEFAULT false, + exec_sessions BOOLEAN DEFAULT false, + microservice_logs BOOLEAN DEFAULT false, + fog_logs BOOLEAN DEFAULT false, + models BOOLEAN DEFAULT false, + runtime_classes BOOLEAN DEFAULT false, + microservice_models BOOLEAN DEFAULT false, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_change_tracking_iofog_uuid ON ChangeTrackings (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogProvisionKeys ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + provisioning_string VARCHAR(100), + expiration_time BIGINT, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_provision_keys_iofogUuid ON FogProvisionKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogVersionCommands ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + version_command VARCHAR(100), + semver VARCHAR(100), + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_version_commands_iofogUuid ON FogVersionCommands (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Tunnels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + username TEXT, + password TEXT, + host TEXT, + remote_port INT, + local_port INT DEFAULT 22, + rsa_key TEXT, + closed BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_tunnels_iofogUuid ON Tunnels (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Microservices ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + config TEXT, + name VARCHAR(255) DEFAULT 'New Microservice', + config_last_updated BIGINT, + rebuild BOOLEAN DEFAULT false, + log_size BIGINT DEFAULT 0, + `delete` BOOLEAN DEFAULT false, + delete_with_cleanup BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + catalog_item_id INT, + registry_id INT DEFAULT 1, + iofog_uuid VARCHAR(36), + application_id INT, + run_as_user TEXT, + platform TEXT, + runtime TEXT, + annotations TEXT, + pid_mode VARCHAR(36), + ipc_mode VARCHAR(36), + schedule INT DEFAULT 50, + cpu_set_cpus TEXT, + memory_limit FLOAT, + is_activated BOOLEAN DEFAULT true, + host_network_mode BOOLEAN DEFAULT false, + is_privileged BOOLEAN DEFAULT false, + nats_rule_id INTEGER, + nats_access BOOLEAN DEFAULT false, + nats_account_id INTEGER, + nats_user_id INTEGER, + nats_creds_secret_name TEXT, + is_controller BOOLEAN DEFAULT false, + run_as_group TEXT, + read_only_root_filesystem BOOLEAN DEFAULT false, + working_dir TEXT, + cpus FLOAT, + memory_reservation FLOAT, + memory_swap FLOAT, + shm_size FLOAT, + sysctls TEXT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservices_catalogItemId ON Microservices (catalog_item_id); +CREATE INDEX idx_microservices_registryId ON Microservices (registry_id); +CREATE INDEX idx_microservices_iofogUuid ON Microservices (iofog_uuid); +CREATE INDEX idx_microservices_applicationId ON Microservices (application_id); + +CREATE TABLE IF NOT EXISTS MicroserviceArgs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cmd TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_args_microserviceUuid ON MicroserviceArgs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceEnvs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + `key` TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + value_from_secret TEXT, + value_from_config_map TEXT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_envs_microserviceUuid ON MicroserviceEnvs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExtraHost ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + template_type TEXT, + name TEXT, + template TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + target_microservice_uuid VARCHAR(36), + target_fog_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_extra_host_microserviceUuid ON MicroserviceExtraHost (microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetMicroserviceUuid ON MicroserviceExtraHost (target_microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetFogUuid ON MicroserviceExtraHost (target_fog_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePorts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + port_internal INT, + port_external INT, + is_udp BOOLEAN, + created_at DATETIME, + updated_at DATETIME, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_port_microserviceUuid ON MicroservicePorts (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + status VARCHAR(255) DEFAULT 'QUEUED', + operating_duration BIGINT DEFAULT 0, + start_time BIGINT DEFAULT 0, + cpu_usage FLOAT DEFAULT 0.000, + memory_usage BIGINT DEFAULT 0, + container_id VARCHAR(255) DEFAULT '', + percentage FLOAT DEFAULT 0.00, + error_message TEXT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + ip_address TEXT, + exec_session_ids TEXT, + health_status TEXT, + pod_id TEXT DEFAULT '', + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_status_microserviceUuid ON MicroserviceStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS VolumeMappings ( + uuid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + host_destination TEXT, + container_destination TEXT, + access_mode TEXT, + type TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mappings_microserviceUuid ON VolumeMappings (microservice_uuid); + +CREATE TABLE IF NOT EXISTS CatalogItemImages ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + container_image TEXT, + catalog_item_id INT, + microservice_uuid VARCHAR(36), + arch_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_image_catalog_item_id ON CatalogItemImages (catalog_item_id); +CREATE INDEX idx_catalog_item_image_microservice_uuid ON CatalogItemImages (microservice_uuid); +CREATE INDEX idx_catalog_item_image_arch_id ON CatalogItemImages (arch_id); + +CREATE TABLE IF NOT EXISTS CatalogItemInputTypes ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_input_type_catalog_item_id ON CatalogItemInputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS CatalogItemOutputTypes ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_output_type_catalog_item_id ON CatalogItemOutputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS Routers ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + is_edge BOOLEAN DEFAULT true, + messaging_port INT DEFAULT 5671, + edge_router_port INT, + inter_router_port INT, + host TEXT, + is_default BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_router_iofogUuid ON Routers (iofog_uuid); + +CREATE TABLE RouterConnections ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + source_router INT, + dest_router INT, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL, + FOREIGN KEY (source_router) REFERENCES Routers(id) ON DELETE CASCADE, + FOREIGN KEY (dest_router) REFERENCES Routers(id) ON DELETE CASCADE +); + +CREATE INDEX idx_routerconnections_sourceRouter ON RouterConnections (source_router); +CREATE INDEX idx_routerconnections_destRouter ON RouterConnections (dest_router); + +CREATE TABLE IF NOT EXISTS Config ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + `key` VARCHAR(255) NOT NULL UNIQUE, + value VARCHAR(255) NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_config_key ON Config (`key`); + +CREATE TABLE IF NOT EXISTS Tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + value VARCHAR(255) UNIQUE NOT NULL +); + +CREATE TABLE IF NOT EXISTS IofogTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_iofogtags_fog_uuid ON IofogTags (fog_uuid); +CREATE INDEX idx_iofogtags_tag_id ON IofogTags (tag_id); + +CREATE TABLE IF NOT EXISTS ApplicationTemplates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL DEFAULT 'new-application', + description VARCHAR(255) DEFAULT '', + schema_version VARCHAR(255) DEFAULT '', + application_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS ApplicationTemplateVariables ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + application_template_id INT NOT NULL, + `key` TEXT, + description VARCHAR(255) DEFAULT '', + default_value VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (application_template_id) REFERENCES ApplicationTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_applicationtemplatevariables_application_template_id ON ApplicationTemplateVariables (application_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCdiDevices ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cdi_devices TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_cdiDevices_microserviceUuid ON MicroserviceCdiDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePubTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceSubTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicepubtags_microservice_uuid ON MicroservicePubTags (microservice_uuid); +CREATE INDEX idx_microservicesubtags_microservice_uuid ON MicroservicesubTags (microservice_uuid); +CREATE INDEX idx_microservicepubtags_tag_id ON MicroservicePubTags (tag_id); +CREATE INDEX idx_microservicesubtags_tag_id ON MicroservicesubTags (tag_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCapAdd ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cap_add TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capAdd_microserviceUuid ON MicroserviceCapAdd (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceCapDrop ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cap_drop TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capDrop_microserviceUuid ON MicroserviceCapDrop (microservice_uuid); + +CREATE TABLE IF NOT EXISTS FogPublicKeys ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + public_key TEXT, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_public_keys_iofogUuid ON FogPublicKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogUsedTokens ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + jti VARCHAR(255) NOT NULL, + iofog_uuid VARCHAR(36), + expiry_time BIGINT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_used_tokens_iofogUuid ON FogUsedTokens (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Secrets ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('Opaque', 'tls')), + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_secrets_name ON Secrets (name); + +CREATE TABLE IF NOT EXISTS Certificates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + subject TEXT NOT NULL, + is_ca BOOLEAN DEFAULT false, + signed_by_id INTEGER, + hosts TEXT, + valid_from DATETIME NOT NULL, + valid_to DATETIME NOT NULL, + serial_number TEXT NOT NULL, + secret_id INTEGER, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (signed_by_id) REFERENCES Certificates (id) ON DELETE SET NULL, + FOREIGN KEY (secret_id) REFERENCES Secrets (id) ON DELETE CASCADE +); + +CREATE INDEX idx_certificates_name ON Certificates (name); +CREATE INDEX idx_certificates_valid_to ON Certificates (valid_to); +CREATE INDEX idx_certificates_is_ca ON Certificates (is_ca); +CREATE INDEX idx_certificates_signed_by_id ON Certificates (signed_by_id); +CREATE INDEX idx_certificates_secret_id ON Certificates (secret_id); + +CREATE TABLE IF NOT EXISTS Services ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + type TEXT NOT NULL, + resource TEXT NOT NULL, + target_port INTEGER NOT NULL, + service_port INTEGER, + k8s_type TEXT, + bridge_port INTEGER, + default_bridge TEXT, + service_endpoint TEXT, + created_at DATETIME, + updated_at DATETIME, + provisioning_status VARCHAR(36) DEFAULT 'pending', + provisioning_error TEXT +); + +CREATE INDEX idx_services_id ON Services (id); +CREATE INDEX idx_services_name ON Services (name); + +CREATE TABLE IF NOT EXISTS ServiceTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + service_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (service_id) REFERENCES Services (id) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_service_tags_service_id ON ServiceTags (service_id); +CREATE INDEX idx_service_tags_tag_id ON ServiceTags (tag_id); + +CREATE TABLE IF NOT EXISTS ConfigMaps ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + immutable BOOLEAN DEFAULT false, + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + use_vault BOOLEAN DEFAULT true +); + +CREATE INDEX idx_config_maps_name ON ConfigMaps (name); + +CREATE TABLE IF NOT EXISTS VolumeMounts ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + config_map_name VARCHAR(255), + secret_name VARCHAR(255), + version INTEGER DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (config_map_name) REFERENCES ConfigMaps (name) ON DELETE CASCADE, + FOREIGN KEY (secret_name) REFERENCES Secrets (name) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mounts_uuid ON VolumeMounts (uuid); +CREATE INDEX idx_volume_mounts_config_map_name ON VolumeMounts (config_map_name); +CREATE INDEX idx_volume_mounts_secret_name ON VolumeMounts (secret_name); + +CREATE TABLE IF NOT EXISTS FogVolumeMounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + volume_mount_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (volume_mount_uuid) REFERENCES VolumeMounts (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_volume_mounts_fog_uuid ON FogVolumeMounts (fog_uuid); +CREATE INDEX idx_fog_volume_mounts_volume_mount_uuid ON FogVolumeMounts (volume_mount_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExecStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + status VARCHAR(255) DEFAULT 'INACTIVE', + exec_session_id VARCHAR(255), + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_status_microservice_uuid ON MicroserviceExecStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceHealthChecks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + test TEXT, + interval FLOAT, + timeout FLOAT, + start_period FLOAT, + start_interval FLOAT, + retries INT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_health_check_microservice_uuid ON MicroserviceHealthChecks (microservice_uuid); + +CREATE TABLE IF NOT EXISTS Events ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + timestamp BIGINT NOT NULL, + event_type VARCHAR(20) NOT NULL, + endpoint_type VARCHAR(10) NOT NULL, + actor_id VARCHAR(255), + method VARCHAR(10), + resource_type VARCHAR(50), + resource_id VARCHAR(255), + endpoint_path TEXT NOT NULL, + ip_address VARCHAR(45), + status VARCHAR(20) NOT NULL, + status_code INTEGER, + status_message TEXT, + request_id VARCHAR(255), + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX IF NOT EXISTS idx_events_timestamp ON Events (timestamp); +CREATE INDEX IF NOT EXISTS idx_events_endpoint_type ON Events (endpoint_type); +CREATE INDEX IF NOT EXISTS idx_events_actor_id ON Events (actor_id); +CREATE INDEX IF NOT EXISTS idx_events_resource_type ON Events (resource_type); +CREATE INDEX IF NOT EXISTS idx_events_status ON Events (status); +CREATE INDEX IF NOT EXISTS idx_events_method ON Events (method); +CREATE INDEX IF NOT EXISTS idx_events_event_type ON Events (event_type); +CREATE INDEX IF NOT EXISTS idx_events_created_at ON Events (created_at); + +CREATE TABLE IF NOT EXISTS MicroserviceLogStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36), + log_session_id TEXT, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_log_status_microservice_uuid ON MicroserviceLogStatuses (microservice_uuid); +CREATE INDEX idx_microservice_log_status_session_id ON MicroserviceLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS MicroserviceExecSessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON MicroserviceExecSessions (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON MicroserviceExecSessions (session_id); + +CREATE TABLE IF NOT EXISTS FogLogStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + iofog_uuid VARCHAR(36), + log_session_id TEXT, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_log_status_iofog_uuid ON FogLogStatuses (iofog_uuid); +CREATE INDEX idx_fog_log_status_session_id ON FogLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS RbacRoles ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + kind TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS RbacRoleRules ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + role_id INTEGER NOT NULL, + api_groups TEXT NOT NULL, + resources TEXT NOT NULL, + verbs TEXT NOT NULL, + resource_names TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (role_id) REFERENCES RbacRoles (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS RbacRoleBindings ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + kind TEXT, + role_ref TEXT NOT NULL, + subjects TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + role_id INTEGER +); + +CREATE TABLE IF NOT EXISTS RbacServiceAccounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + role_ref TEXT, + role_id INTEGER, + microservice_uuid VARCHAR(36), + application_id INTEGER, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_rbac_role_rules_role_id ON RbacRoleRules (role_id); +CREATE INDEX idx_rbac_roles_name ON RbacRoles (name); +CREATE INDEX idx_rbac_role_bindings_name ON RbacRoleBindings (name); +CREATE INDEX idx_rbac_service_accounts_name ON RbacServiceAccounts (name); + +CREATE TABLE IF NOT EXISTS RbacCacheVersion ( + id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1), + version INTEGER NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_rbac_role_bindings_role_id ON RbacRoleBindings (role_id); + +CREATE INDEX idx_rbac_service_accounts_role_id ON RbacServiceAccounts (role_id); +CREATE UNIQUE INDEX idx_rbac_service_accounts_microservice_uuid_unique ON RbacServiceAccounts (microservice_uuid) WHERE microservice_uuid IS NOT NULL; +CREATE UNIQUE INDEX idx_rbac_service_accounts_application_id_name_unique ON RbacServiceAccounts (application_id, name); + +CREATE TABLE IF NOT EXISTS ClusterControllers ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + host VARCHAR(255), + process_id INTEGER, + last_heartbeat DATETIME, + is_active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_cluster_controllers_uuid ON ClusterControllers (uuid); +CREATE INDEX idx_cluster_controllers_host ON ClusterControllers (host); +CREATE INDEX idx_cluster_controllers_active ON ClusterControllers (is_active, last_heartbeat); + +CREATE TABLE IF NOT EXISTS NatsOperators ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsAccounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + is_system BOOLEAN DEFAULT false, + is_leaf_system BOOLEAN DEFAULT false, + operator_id INTEGER NOT NULL, + application_id INTEGER, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (operator_id) REFERENCES NatsOperators (id) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsUsers ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + creds_secret_name TEXT NOT NULL, + is_bearer BOOLEAN DEFAULT false, + account_id INTEGER NOT NULL, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + nats_user_rule_id INTEGER, + FOREIGN KEY (account_id) REFERENCES NatsAccounts (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS NatsInstances ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + iofog_uuid VARCHAR(36), + is_leaf BOOLEAN DEFAULT true, + is_hub BOOLEAN DEFAULT false, + host TEXT, + server_port INTEGER, + leaf_port INTEGER, + cluster_port INTEGER, + mqtt_port INTEGER, + http_port INTEGER, + configmap_name TEXT, + jwt_dir_mount_name TEXT, + cert_secret_name TEXT, + js_storage_size TEXT, + js_memory_store_size TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsConnections ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + source_nats INTEGER NOT NULL, + dest_nats INTEGER NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (source_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE, + FOREIGN KEY (dest_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsReconcileTasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + reason TEXT NOT NULL, + application_id INTEGER, + account_rule_id INTEGER, + user_rule_id INTEGER, + fog_uuids TEXT, + status TEXT NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsAccountRules ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + info_url TEXT, + max_connections INTEGER, + max_leaf_node_connections INTEGER, + max_data BIGINT, + max_exports INTEGER, + max_imports INTEGER, + max_msg_payload INTEGER, + max_subscriptions INTEGER, + exports_allow_wildcards BOOLEAN DEFAULT true, + disallow_bearer BOOLEAN, + response_permissions TEXT, + resp_max INTEGER, + resp_ttl BIGINT, + imports TEXT, + exports TEXT, + mem_storage BIGINT, + disk_storage BIGINT, + streams INTEGER, + consumer INTEGER, + max_ack_pending INTEGER, + mem_max_stream_bytes BIGINT, + disk_max_stream_bytes BIGINT, + max_bytes_required BOOLEAN, + tiered_limits TEXT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsUserRules ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + max_subscriptions INTEGER, + max_payload INTEGER, + max_data BIGINT, + bearer_token BOOLEAN DEFAULT false, + proxy_required BOOLEAN, + allowed_connection_types TEXT, + src TEXT, + times TEXT, + times_location TEXT, + resp_max INTEGER, + resp_ttl BIGINT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + tags TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE UNIQUE INDEX idx_nats_accounts_application_id_unique ON NatsAccounts (application_id) WHERE application_id IS NOT NULL; +CREATE INDEX idx_nats_accounts_application_id ON NatsAccounts (application_id); +CREATE UNIQUE INDEX idx_nats_users_account_id_name ON NatsUsers (account_id, name); +CREATE INDEX idx_nats_users_account_id ON NatsUsers (account_id); +CREATE INDEX idx_nats_users_microservice_uuid ON NatsUsers (microservice_uuid); +CREATE INDEX idx_nats_users_nats_user_rule_id ON NatsUsers (nats_user_rule_id); +CREATE UNIQUE INDEX idx_nats_instances_iofog_uuid_unique ON NatsInstances (iofog_uuid); +CREATE INDEX idx_nats_instances_iofog_uuid ON NatsInstances (iofog_uuid); +CREATE UNIQUE INDEX idx_nats_connections_source_dest_unique ON NatsConnections (source_nats, dest_nats); +CREATE INDEX idx_nats_connections_source_nats ON NatsConnections (source_nats); +CREATE INDEX idx_nats_connections_dest_nats ON NatsConnections (dest_nats); +CREATE INDEX idx_nats_account_rules_name ON NatsAccountRules (name); +CREATE INDEX idx_nats_user_rules_name ON NatsUserRules (name); +CREATE INDEX idx_nats_reconcile_tasks_status_claimed ON NatsReconcileTasks (status, claimed_at); + +CREATE INDEX idx_applications_nats_rule_id ON Applications (nats_rule_id); +CREATE INDEX idx_microservices_nats_rule_id ON Microservices (nats_rule_id); +CREATE INDEX idx_microservices_nats_account_id ON Microservices (nats_account_id); +CREATE INDEX idx_microservices_nats_user_id ON Microservices (nats_user_id); + +CREATE TABLE IF NOT EXISTS AuthUsers ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + password_history_hashes TEXT, + must_change_password BOOLEAN DEFAULT false, + is_bootstrap BOOLEAN DEFAULT false, + failed_attempts INT DEFAULT 0, + locked_until DATETIME, + deleted_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_users_email ON AuthUsers (email); +CREATE INDEX idx_auth_users_deleted_at ON AuthUsers (deleted_at); + +CREATE TABLE IF NOT EXISTS AuthGroups ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + is_system BOOLEAN DEFAULT false, + mfa_required BOOLEAN NOT NULL DEFAULT false, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthUserGroups ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + user_id VARCHAR(36) NOT NULL, + group_id INT NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE, + FOREIGN KEY (group_id) REFERENCES AuthGroups (id) ON DELETE CASCADE, + UNIQUE (user_id, group_id) +); + +CREATE INDEX idx_auth_user_groups_user_id ON AuthUserGroups (user_id); +CREATE INDEX idx_auth_user_groups_group_id ON AuthUserGroups (group_id); + +CREATE TABLE IF NOT EXISTS AuthMfa ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + user_id VARCHAR(36) NOT NULL UNIQUE, + totp_secret_encrypted TEXT, + enabled BOOLEAN DEFAULT false, + recovery_codes_hash TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_mfa_user_id ON AuthMfa (user_id); + +CREATE TABLE IF NOT EXISTS AuthPasswordResetSessions ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_password_reset_sessions_user_id ON AuthPasswordResetSessions (user_id); +CREATE INDEX idx_auth_password_reset_sessions_expires_at ON AuthPasswordResetSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthRefreshTokens ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + token_hash VARCHAR(255) NOT NULL, + user_id VARCHAR(36) NOT NULL, + family_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + revoked BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_refresh_tokens_token_hash ON AuthRefreshTokens (token_hash); +CREATE INDEX idx_auth_refresh_tokens_user_id ON AuthRefreshTokens (user_id); +CREATE INDEX idx_auth_refresh_tokens_family_id ON AuthRefreshTokens (family_id); +CREATE INDEX idx_auth_refresh_tokens_expires_at ON AuthRefreshTokens (expires_at); + +CREATE TABLE IF NOT EXISTS AuthOidcKeys ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + kid VARCHAR(255) NOT NULL UNIQUE, + key_material_encrypted TEXT, + vault_ref TEXT, + active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_oidc_keys_active ON AuthOidcKeys (active); + +CREATE TABLE IF NOT EXISTS AuthOidcClients ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + client_id VARCHAR(255) NOT NULL UNIQUE, + secret_ref TEXT, + client_type VARCHAR(32) NOT NULL DEFAULT 'confidential', + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthOidcProviderStates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + model VARCHAR(64) NOT NULL, + record_id VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME, + grant_id VARCHAR(255), + uid VARCHAR(255), + user_code VARCHAR(255), + consumed BOOLEAN DEFAULT false, + consumed_at DATETIME, + created_at DATETIME, + updated_at DATETIME, + UNIQUE (model, record_id) +); + +CREATE INDEX idx_auth_oidc_provider_states_grant_id ON AuthOidcProviderStates (grant_id); +CREATE INDEX idx_auth_oidc_provider_states_uid ON AuthOidcProviderStates (uid); +CREATE INDEX idx_auth_oidc_provider_states_user_code ON AuthOidcProviderStates (user_code); +CREATE INDEX idx_auth_oidc_provider_states_expires_at ON AuthOidcProviderStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBffSessions ( + sid VARCHAR(255) PRIMARY KEY NOT NULL, + data TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_bff_sessions_expires_at ON AuthBffSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthInteractionStates ( + uid VARCHAR(255) PRIMARY KEY NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_interaction_states_expires_at ON AuthInteractionStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBootstrapMeta ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + completed_at DATETIME, + bootstrap_admin_user_id VARCHAR(36), + session_secret_ref TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (bootstrap_admin_user_id) REFERENCES AuthUsers (id) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS AuthPolicy ( + id INT PRIMARY KEY NOT NULL CHECK (id = 1), + min_password_length INT DEFAULT 12, + require_uppercase BOOLEAN DEFAULT true, + require_lowercase BOOLEAN DEFAULT true, + require_digit BOOLEAN DEFAULT true, + password_max_age_days INT DEFAULT 0, + password_history_count INT DEFAULT 5, + max_failed_attempts INT DEFAULT 5, + lockout_duration_minutes INT DEFAULT 15, + access_token_ttl_seconds INT DEFAULT 900, + refresh_token_ttl_seconds INT DEFAULT 3600, + refresh_rotation BOOLEAN DEFAULT true, + max_concurrent_sessions INT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS FogPlatformSpecs ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + spec_json TEXT NOT NULL, + generation INTEGER NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformStatuses ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + observed_generation INTEGER NOT NULL DEFAULT 0, + phase VARCHAR(32) NOT NULL DEFAULT 'Pending', + last_error TEXT, + last_transition_at DATETIME, + conditions_json TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformReconcileTasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36) NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_generation INTEGER, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX idx_fog_platform_reconcile_tasks_active_fog_uuid ON FogPlatformReconcileTasks (fog_uuid) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_fog_platform_reconcile_tasks_status_claimed ON FogPlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_fog_platform_reconcile_tasks_next_attempt ON FogPlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ServicePlatformReconcileTasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + service_name TEXT NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_snapshot TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE UNIQUE INDEX idx_service_platform_reconcile_tasks_active_service_name ON ServicePlatformReconcileTasks (service_name) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_service_platform_reconcile_tasks_status_claimed ON ServicePlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_service_platform_reconcile_tasks_next_attempt ON ServicePlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ReconcileOutbox ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + kind VARCHAR(32) NOT NULL, + payload TEXT NOT NULL, + idempotency_key VARCHAR(255) NOT NULL UNIQUE, + created_at DATETIME, + processed_at DATETIME, + last_error TEXT +); + +CREATE INDEX idx_reconcile_outbox_unprocessed ON ReconcileOutbox (processed_at, id); + +CREATE TABLE IF NOT EXISTS HubRouterConfigLocks ( + id INTEGER PRIMARY KEY NOT NULL CHECK (id = 1), + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON Models (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON MicroserviceModelItems (name); diff --git a/src/data/models/index.js b/src/data/models/index.js index ccfa2a9c..e3685109 100644 --- a/src/data/models/index.js +++ b/src/data/models/index.js @@ -63,16 +63,13 @@ db.initDB = async (isStart) => { if (databaseProvider instanceof require('../providers/sqlite')) { const sqliteDbPath = databaseProvider.sequelize.options.storage logger.info('Running SQLite database migrations and seeders...') - await databaseProvider.runMigrationSQLite(sqliteDbPath) - await databaseProvider.runSeederSQLite(sqliteDbPath) + await databaseProvider.runVersionChainSQLite(sqliteDbPath) } else if (databaseProvider instanceof require('../providers/mysql')) { logger.info('Running MySQL database migrations and seeders...') - await databaseProvider.runMigrationMySQL(databaseProvider.sequelize) - await databaseProvider.runSeederMySQL(databaseProvider.sequelize) + await databaseProvider.runVersionChainMySQL(databaseProvider.sequelize) } else if (databaseProvider instanceof require('../providers/postgres')) { logger.info('Running PostgreSQL database migrations and seeders...') - await databaseProvider.runMigrationPostgres(databaseProvider.sequelize) - await databaseProvider.runSeederPostgres(databaseProvider.sequelize) + await databaseProvider.runVersionChainPostgres(databaseProvider.sequelize) } // Initialize RBAC cache version if it doesn't exist diff --git a/src/data/providers/database-provider.js b/src/data/providers/database-provider.js index 331f60ea..eaf03fc2 100644 --- a/src/data/providers/database-provider.js +++ b/src/data/providers/database-provider.js @@ -21,6 +21,133 @@ function sqliteClose (db) { }) } +const SCHEMA_VERSIONS = ['3.8.0', '3.9.0'] +const BASELINE_VERSION = '3.9.0' + +function compareVersions (left, right) { + const leftParts = String(left || '0').split('.').map((part) => parseInt(part, 10) || 0) + const rightParts = String(right || '0').split('.').map((part) => parseInt(part, 10) || 0) + const length = Math.max(leftParts.length, rightParts.length) + for (let i = 0; i < length; i++) { + const delta = (leftParts[i] || 0) - (rightParts[i] || 0) + if (delta !== 0) { + return delta + } + } + return 0 +} + +function isVersionLessThan (current, target) { + return current == null || compareVersions(current, target) < 0 +} + +function versionsToApply (untilVersion) { + if (!untilVersion) { + return SCHEMA_VERSIONS.slice() + } + return SCHEMA_VERSIONS.filter((version) => compareVersions(version, untilVersion) <= 0) +} + +function shouldApplyBaseline (currentSchemaVersion, untilVersion) { + return currentSchemaVersion == null && untilVersion == null +} + +function splitSqlStatements (sql) { + return sql.split(';').map((query) => query.trim()).filter(Boolean) +} + +function isTransactionControl (query) { + const normalized = query.replace(/;$/, '').trim().toUpperCase() + return normalized === 'START TRANSACTION' || + normalized === 'BEGIN' || + normalized === 'BEGIN TRANSACTION' || + normalized === 'COMMIT' || + normalized === 'ROLLBACK' +} + +function migrationSqlPath (provider, version) { + const relative = { + sqlite: `sqlite/db_migration_sqlite_v${version}.sql`, + mysql: `mysql/db_migration_mysql_v${version}.sql`, + postgres: `postgres/db_migration_pg_v${version}.sql` + }[provider] + return path.resolve(__dirname, '../migrations', relative) +} + +function seederSqlPath (provider, version) { + const relative = { + sqlite: `sqlite/db_seeder_sqlite_v${version}.sql`, + mysql: `mysql/db_seeder_mysql_v${version}.sql`, + postgres: `postgres/db_seeder_pg_v${version}.sql` + }[provider] + return path.resolve(__dirname, '../seeders', relative) +} + +function baselineMigrationSqlPath (provider) { + const relative = { + sqlite: 'sqlite/db_migration_sqlite_baseline_v3.9.0.sql', + mysql: 'mysql/db_migration_mysql_baseline_v3.9.0.sql', + postgres: 'postgres/db_migration_pg_baseline_v3.9.0.sql' + }[provider] + return path.resolve(__dirname, '../migrations', relative) +} + +function baselineSeederSqlPath (provider) { + const relative = { + sqlite: 'sqlite/db_seeder_sqlite_baseline_v3.9.0.sql', + mysql: 'mysql/db_seeder_mysql_baseline_v3.9.0.sql', + postgres: 'postgres/db_seeder_pg_baseline_v3.9.0.sql' + }[provider] + return path.resolve(__dirname, '../seeders', relative) +} + +function readSqlFile (filePath) { + if (!fs.existsSync(filePath)) { + logger.error(`SQL file not found: ${filePath}`) + throw new Error('SQL file not found') + } + return fs.readFileSync(filePath).toString() +} + +function isIgnorableSqliteError (err) { + const message = (err && err.message) || '' + return message.includes('already exists') || message.includes('duplicate') +} + +function isIgnorableMysqlError (err) { + const errorToCheck = (err && err.parent) || err || {} + return errorToCheck.code === 'ER_TABLE_EXISTS_ERROR' || + errorToCheck.code === 'ER_DUP_FIELDNAME' || + errorToCheck.code === 'ER_DUP_KEYNAME' || + errorToCheck.code === 'ER_BLOB_KEY_WITHOUT_LENGTH' || + errorToCheck.code === 'ER_CANT_DROP_FIELD_OR_KEY' || + errorToCheck.code === 'ER_DUP_ENTRY' || + errorToCheck.code === 'ER_DUP_KEY' || + errorToCheck.code === 'duplicate_key' || + errorToCheck.code === 'already_exists' || + errorToCheck.errno === 1091 || + errorToCheck.errno === 1061 || + errorToCheck.errno === 1170 +} + +function isIgnorablePostgresError (err) { + const errorToCheck = (err && err.parent) || err || {} + const message = errorToCheck.message || '' + return errorToCheck.code === '42P07' || + errorToCheck.code === '42701' || + errorToCheck.code === '42P06' || + errorToCheck.code === '23505' || + errorToCheck.code === '23503' || + errorToCheck.code === '42P01' || + errorToCheck.code === '42703' || + errorToCheck.code === '42P16' || + errorToCheck.code === '42P17' || + errorToCheck.code === '42P18' || + message.includes('already exists') || + message.includes('duplicate key') || + message.includes('does not exist') +} + class DatabaseProvider { constructor () { this.basename = path.basename(__filename) @@ -271,375 +398,418 @@ class DatabaseProvider { } } - // SQLite migration — greenfield v3.8.0 (see src/data/migrations/README.md) - async runMigrationSQLite (dbName) { - const migrationSqlPath = path.resolve(__dirname, '../migrations/sqlite/db_migration_sqlite_v3.8.0.sql') - const migrationVersion = '3.8.0' - - if (!fs.existsSync(migrationSqlPath)) { - logger.error(`Migration file not found: ${migrationSqlPath}`) - throw new Error('Migration file not found') + async applySqliteStatements (db, sql, { ignoreDuplicate } = { ignoreDuplicate: true }) { + for (const raw of splitSqlStatements(sql)) { + if (isTransactionControl(raw)) { + continue + } + const query = raw + ';' + try { + await sqliteRun(db, query) + } catch (err) { + if (ignoreDuplicate && isIgnorableSqliteError(err)) { + logger.warn(`Ignored error: ${err.message}`) + } else { + throw err + } + } } + } - const migrationSql = fs.readFileSync(migrationSqlPath).toString() - const dataArr = migrationSql.split(';') + async applySqliteMigrationIfNeeded (db, version) { + const currentVersion = await this.checkMigrationVersion(db, 'sqlite') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`SQLite schema ${currentVersion || 'none'} already includes ${version}, skipping migration`) + return + } - const db = new sqlite3.Database(dbName, (err) => { - if (err) { - logger.error(err.message) - throw err + const sql = readSqlFile(migrationSqlPath('sqlite', version)) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, sql) + await this.updateMigrationVersion(db, version, 'sqlite') + await sqliteRun(db, 'COMMIT') + logger.info(`SQLite migration ${version} completed successfully.`) + } catch (err) { + try { + await sqliteRun(db, 'ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } - logger.info('Connected to the SQLite database for migration.') - }) + logger.error(`SQLite migration ${version} failed:`, err) + throw err + } + } - try { - await this.createSchemaVersionTable(db, 'sqlite') - const currentVersion = await this.checkMigrationVersion(db, 'sqlite') + async applySqliteSeederIfNeeded (db, version) { + const currentVersion = await this.checkSeederVersion(db, 'sqlite') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`SQLite seeder ${currentVersion || 'none'} already includes ${version}, skipping seeder`) + return + } - if (currentVersion === migrationVersion) { - logger.info('Migration already up to date, skipping...') - return + const sql = readSqlFile(seederSqlPath('sqlite', version)) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, sql) + await this.updateSeederVersion(db, version, 'sqlite') + await sqliteRun(db, 'COMMIT') + logger.info(`SQLite seeder ${version} completed successfully.`) + } catch (err) { + try { + await sqliteRun(db, 'ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`SQLite seeder ${version} failed:`, err) + throw err + } + } - await sqliteRun(db, 'PRAGMA foreign_keys=OFF') - await sqliteRun(db, 'BEGIN TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await sqliteRun(db, query) - } catch (err) { - if (err.message.includes('already exists') || err.message.includes('duplicate')) { - logger.warn(`Ignored error: ${err.message}`) - } else { - throw err - } - } - } + async applySqliteBaseline (db) { + logger.info(`Applying empty-database baseline schema ${BASELINE_VERSION}`) + const migrationSql = readSqlFile(baselineMigrationSqlPath('sqlite')) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, migrationSql) + await this.updateMigrationVersion(db, BASELINE_VERSION, 'sqlite') + await sqliteRun(db, 'COMMIT') + logger.info(`SQLite baseline migration ${BASELINE_VERSION} completed successfully.`) + } catch (err) { + try { + await sqliteRun(db, 'ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`SQLite baseline migration ${BASELINE_VERSION} failed:`, err) + throw err + } - await this.updateMigrationVersion(db, migrationVersion, 'sqlite') + logger.info(`Applying empty-database baseline seed ${BASELINE_VERSION}`) + const seederSql = readSqlFile(baselineSeederSqlPath('sqlite')) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, seederSql) + await this.updateSeederVersion(db, BASELINE_VERSION, 'sqlite') await sqliteRun(db, 'COMMIT') - logger.info('Migration completed successfully.') + logger.info(`SQLite baseline seeder ${BASELINE_VERSION} completed successfully.`) } catch (err) { try { await sqliteRun(db, 'ROLLBACK') } catch (rollbackErr) { // No active transaction to roll back. } - logger.error('Migration failed:', err) + logger.error(`SQLite baseline seeder ${BASELINE_VERSION} failed:`, err) throw err + } + } + + async runVersionChainSQLite (dbName, options = {}) { + const db = new sqlite3.Database(dbName, (err) => { + if (err) { + logger.error(err.message) + throw err + } + logger.info('Connected to the SQLite database for schema updates.') + }) + + try { + await this.createSchemaVersionTable(db, 'sqlite') + await sqliteRun(db, 'PRAGMA foreign_keys=OFF') + const currentVersion = await this.checkMigrationVersion(db, 'sqlite') + if (shouldApplyBaseline(currentVersion, options.untilVersion)) { + await this.applySqliteBaseline(db) + return + } + for (const version of versionsToApply(options.untilVersion)) { + await this.applySqliteMigrationIfNeeded(db, version) + await this.applySqliteSeederIfNeeded(db, version) + } } finally { try { await sqliteClose(db) - logger.info('Database connection closed after migration.') + logger.info('Database connection closed after schema updates.') } catch (closeErr) { logger.error('Error closing database connection:', closeErr.message) } } } - // MySQL migration — greenfield v3.8.0 (see src/data/migrations/README.md) - async runMigrationMySQL (db) { - const migrationSqlPath = path.resolve(__dirname, '../migrations/mysql/db_migration_mysql_v3.8.0.sql') - const migrationVersion = '3.8.0' - - if (!fs.existsSync(migrationSqlPath)) { - logger.error(`Migration file not found: ${migrationSqlPath}`) - throw new Error('Migration file not found') - } - - const migrationSql = fs.readFileSync(migrationSqlPath).toString() - const dataArr = migrationSql.split(';') - - try { - await this.createSchemaVersionTable(db, 'mysql') - const currentVersion = await this.checkMigrationVersion(db, 'mysql') - - if (currentVersion === migrationVersion) { - logger.info('Migration already up to date, skipping...') - return + async applyMysqlStatements (db, sql) { + for (const raw of splitSqlStatements(sql)) { + if (isTransactionControl(raw)) { + continue } - - await db.query('START TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - const errorToCheck = err.parent || err - if (errorToCheck.code === 'ER_TABLE_EXISTS_ERROR' || - errorToCheck.code === 'ER_DUP_FIELDNAME' || - errorToCheck.code === 'ER_DUP_KEYNAME' || - errorToCheck.code === 'ER_BLOB_KEY_WITHOUT_LENGTH' || - errorToCheck.code === 'ER_CANT_DROP_FIELD_OR_KEY' || - errorToCheck.code === 'duplicate_key' || - errorToCheck.code === 'already_exists' || - errorToCheck.errno === 1091 || - errorToCheck.errno === 1061 || - errorToCheck.errno === 1170) { - logger.warn(`Ignored MySQL error: ${errorToCheck.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } + const query = raw + ';' + try { + await db.query(query) + } catch (err) { + const errorToCheck = err.parent || err + if (errorToCheck.code === '25P02') { + throw err + } + if (isIgnorableMysqlError(err)) { + logger.warn(`Ignored MySQL error: ${errorToCheck.message}`) + } else { + throw err } } + } + } - await this.updateMigrationVersion(db, migrationVersion, 'mysql') + async applyMysqlMigrationIfNeeded (db, version) { + const currentVersion = await this.checkMigrationVersion(db, 'mysql') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`MySQL schema ${currentVersion || 'none'} already includes ${version}, skipping migration`) + return + } + + const sql = readSqlFile(migrationSqlPath('mysql', version)) + await db.query('START TRANSACTION') + try { + await this.applyMysqlStatements(db, sql) + await this.updateMigrationVersion(db, version, 'mysql') await db.query('COMMIT') - logger.info('Migration completed successfully.') + logger.info(`MySQL migration ${version} completed successfully.`) } catch (err) { - await db.query('ROLLBACK') - logger.error('Migration failed:', err) + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. + } + logger.error(`MySQL migration ${version} failed:`, err) throw err } } - // PostgreSQL migration — greenfield v3.8.0 (see src/data/migrations/README.md) - async runMigrationPostgres (db) { - const migrationSqlPath = path.resolve(__dirname, '../migrations/postgres/db_migration_pg_v3.8.0.sql') - const migrationVersion = '3.8.0' - - if (!fs.existsSync(migrationSqlPath)) { - logger.error(`Migration file not found: ${migrationSqlPath}`) - throw new Error('Migration file not found') + async applyMysqlSeederIfNeeded (db, version) { + const currentVersion = await this.checkSeederVersion(db, 'mysql') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`MySQL seeder ${currentVersion || 'none'} already includes ${version}, skipping seeder`) + return } - const migrationSql = fs.readFileSync(migrationSqlPath).toString() - const dataArr = migrationSql.split(';') - + const sql = readSqlFile(seederSqlPath('mysql', version)) + await db.query('START TRANSACTION') try { - await this.createSchemaVersionTable(db, 'postgres') - const currentVersion = await this.checkMigrationVersion(db, 'postgres') - - if (currentVersion === migrationVersion) { - logger.info('Migration already up to date, skipping...') - return + await this.applyMysqlStatements(db, sql) + await this.updateSeederVersion(db, version, 'mysql') + await db.query('COMMIT') + logger.info(`MySQL seeder ${version} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`MySQL seeder ${version} failed:`, err) + throw err + } + } - await db.query('BEGIN') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - const errorToCheck = err.parent || err - - if (errorToCheck.code === '25P02') { - logger.warn('Transaction aborted, rolling back and starting new transaction...') - await db.query('ROLLBACK') - await db.query('BEGIN') - continue - } - - if (errorToCheck.code === '42P07' || - errorToCheck.code === '42701' || - errorToCheck.code === '42P06' || - errorToCheck.code === '23505' || - errorToCheck.code === '23503' || - errorToCheck.code === '42P01' || - errorToCheck.code === '42703' || - errorToCheck.code === '42P16' || - errorToCheck.code === '42P17' || - errorToCheck.code === '42P18' || - (errorToCheck.message && ( - errorToCheck.message.includes('already exists') || - errorToCheck.message.includes('duplicate key') || - errorToCheck.message.includes('relation') - ))) { - logger.warn(`Ignored PostgreSQL error: ${errorToCheck.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } - } + async applyMysqlBaseline (db) { + logger.info(`Applying empty-database baseline schema ${BASELINE_VERSION}`) + const migrationSql = readSqlFile(baselineMigrationSqlPath('mysql')) + await db.query('START TRANSACTION') + try { + await this.applyMysqlStatements(db, migrationSql) + await this.updateMigrationVersion(db, BASELINE_VERSION, 'mysql') + await db.query('COMMIT') + logger.info(`MySQL baseline migration ${BASELINE_VERSION} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`MySQL baseline migration ${BASELINE_VERSION} failed:`, err) + throw err + } - await this.updateMigrationVersion(db, migrationVersion, 'postgres') + logger.info(`Applying empty-database baseline seed ${BASELINE_VERSION}`) + const seederSql = readSqlFile(baselineSeederSqlPath('mysql')) + await db.query('START TRANSACTION') + try { + await this.applyMysqlStatements(db, seederSql) + await this.updateSeederVersion(db, BASELINE_VERSION, 'mysql') await db.query('COMMIT') - logger.info('Migration completed successfully.') + logger.info(`MySQL baseline seeder ${BASELINE_VERSION} completed successfully.`) } catch (err) { - await db.query('ROLLBACK') - logger.error('Migration failed:', err) + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. + } + logger.error(`MySQL baseline seeder ${BASELINE_VERSION} failed:`, err) throw err } } - // SQLite seeder — greenfield v3.8.0 (see src/data/migrations/README.md) - async runSeederSQLite (dbName) { - const seederSqlPath = path.resolve(__dirname, '../seeders/sqlite/db_seeder_sqlite_v3.8.0.sql') - const seederVersion = '3.8.0' - - if (!fs.existsSync(seederSqlPath)) { - logger.error(`Seeder file not found: ${seederSqlPath}`) - throw new Error('Seeder file not found') + async runVersionChainMySQL (db, options = {}) { + await this.createSchemaVersionTable(db, 'mysql') + const currentVersion = await this.checkMigrationVersion(db, 'mysql') + if (shouldApplyBaseline(currentVersion, options.untilVersion)) { + await this.applyMysqlBaseline(db) + return } + for (const version of versionsToApply(options.untilVersion)) { + await this.applyMysqlMigrationIfNeeded(db, version) + await this.applyMysqlSeederIfNeeded(db, version) + } + } - const seederSql = fs.readFileSync(seederSqlPath).toString() - const dataArr = seederSql.split(';') - - const db = new sqlite3.Database(dbName, (err) => { - if (err) { - logger.error(err.message) - throw err - } - logger.info('Connected to the SQLite database for seeding.') - }) - - try { - const currentVersion = await this.checkSeederVersion(db, 'sqlite') - - if (currentVersion === seederVersion) { - logger.info('Seeder already up to date, skipping...') - return + async applyPostgresStatements (db, sql) { + for (const raw of splitSqlStatements(sql)) { + if (isTransactionControl(raw)) { + continue } - - await sqliteRun(db, 'PRAGMA foreign_keys=OFF') - await sqliteRun(db, 'BEGIN TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await sqliteRun(db, query) - } catch (err) { - if (err.message.includes('already exists') || err.message.includes('duplicate')) { - logger.warn(`Ignored error: ${err.message}`) - } else { - throw err - } - } + const query = raw + ';' + try { + await db.query(query) + } catch (err) { + const errorToCheck = err.parent || err + if (errorToCheck.code === '25P02') { + logger.warn('Transaction aborted, rolling back and starting new transaction...') + await db.query('ROLLBACK') + await db.query('BEGIN') + continue + } + if (isIgnorablePostgresError(err)) { + logger.warn(`Ignored PostgreSQL error: ${errorToCheck.message}`) + } else { + throw err } } + } + } - await this.updateSeederVersion(db, seederVersion, 'sqlite') - await sqliteRun(db, 'COMMIT') - logger.info('Seeding completed successfully.') + async applyPostgresMigrationIfNeeded (db, version) { + const currentVersion = await this.checkMigrationVersion(db, 'postgres') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`PostgreSQL schema ${currentVersion || 'none'} already includes ${version}, skipping migration`) + return + } + + const sql = readSqlFile(migrationSqlPath('postgres', version)) + await db.query('BEGIN') + try { + await this.applyPostgresStatements(db, sql) + await this.updateMigrationVersion(db, version, 'postgres') + await db.query('COMMIT') + logger.info(`PostgreSQL migration ${version} completed successfully.`) } catch (err) { try { - await sqliteRun(db, 'ROLLBACK') + await db.query('ROLLBACK') } catch (rollbackErr) { // No active transaction to roll back. } - logger.error('Seeding failed:', err) + logger.error(`PostgreSQL migration ${version} failed:`, err) throw err - } finally { - try { - await sqliteClose(db) - logger.info('Database connection closed after seeding.') - } catch (closeErr) { - logger.error('Error closing database connection:', closeErr.message) - } } } - // MySQL seeder — greenfield v3.8.0 (see src/data/migrations/README.md) - async runSeederMySQL (db) { - const seederSqlPath = path.resolve(__dirname, '../seeders/mysql/db_seeder_mysql_v3.8.0.sql') - const seederVersion = '3.8.0' - - if (!fs.existsSync(seederSqlPath)) { - logger.error(`Seeder file not found: ${seederSqlPath}`) - throw new Error('Seeder file not found') + async applyPostgresSeederIfNeeded (db, version) { + const currentVersion = await this.checkSeederVersion(db, 'postgres') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`PostgreSQL seeder ${currentVersion || 'none'} already includes ${version}, skipping seeder`) + return } - const seederSql = fs.readFileSync(seederSqlPath).toString() - const dataArr = seederSql.split(';') - + const sql = readSqlFile(seederSqlPath('postgres', version)) + await db.query('BEGIN') try { - const currentVersion = await this.checkSeederVersion(db, 'mysql') - - if (currentVersion === seederVersion) { - logger.info('Seeder already up to date, skipping...') - return + await this.applyPostgresStatements(db, sql) + await this.updateSeederVersion(db, version, 'postgres') + await db.query('COMMIT') + logger.info(`PostgreSQL seeder ${version} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`PostgreSQL seeder ${version} failed:`, err) + throw err + } + } - await db.query('START TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - if (err.code === 'ER_DUP_ENTRY' || - err.code === 'ER_DUP_KEY') { - logger.warn(`Ignored MySQL error: ${err.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } - } + async applyPostgresBaseline (db) { + logger.info(`Applying empty-database baseline schema ${BASELINE_VERSION}`) + const migrationSql = readSqlFile(baselineMigrationSqlPath('postgres')) + await db.query('BEGIN') + try { + await this.applyPostgresStatements(db, migrationSql) + await this.updateMigrationVersion(db, BASELINE_VERSION, 'postgres') + await db.query('COMMIT') + logger.info(`PostgreSQL baseline migration ${BASELINE_VERSION} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`PostgreSQL baseline migration ${BASELINE_VERSION} failed:`, err) + throw err + } - await this.updateSeederVersion(db, seederVersion, 'mysql') + logger.info(`Applying empty-database baseline seed ${BASELINE_VERSION}`) + const seederSql = readSqlFile(baselineSeederSqlPath('postgres')) + await db.query('BEGIN') + try { + await this.applyPostgresStatements(db, seederSql) + await this.updateSeederVersion(db, BASELINE_VERSION, 'postgres') await db.query('COMMIT') - logger.info('Seeding completed successfully.') + logger.info(`PostgreSQL baseline seeder ${BASELINE_VERSION} completed successfully.`) } catch (err) { - await db.query('ROLLBACK') - logger.error('Seeding failed:', err) + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. + } + logger.error(`PostgreSQL baseline seeder ${BASELINE_VERSION} failed:`, err) throw err } } - // PostgreSQL seeder — greenfield v3.8.0 (see src/data/migrations/README.md) - async runSeederPostgres (db) { - const seederSqlPath = path.resolve(__dirname, '../seeders/postgres/db_seeder_pg_v3.8.0.sql') - const seederVersion = '3.8.0' - - if (!fs.existsSync(seederSqlPath)) { - logger.error(`Seeder file not found: ${seederSqlPath}`) - throw new Error('Seeder file not found') + async runVersionChainPostgres (db, options = {}) { + await this.createSchemaVersionTable(db, 'postgres') + const currentVersion = await this.checkMigrationVersion(db, 'postgres') + if (shouldApplyBaseline(currentVersion, options.untilVersion)) { + await this.applyPostgresBaseline(db) + return + } + for (const version of versionsToApply(options.untilVersion)) { + await this.applyPostgresMigrationIfNeeded(db, version) + await this.applyPostgresSeederIfNeeded(db, version) } + } - const seederSql = fs.readFileSync(seederSqlPath).toString() - const dataArr = seederSql.split(';') + async runMigrationSQLite (dbName, options) { + return this.runVersionChainSQLite(dbName, options) + } - try { - const currentVersion = await this.checkSeederVersion(db, 'postgres') + async runMigrationMySQL (db, options) { + return this.runVersionChainMySQL(db, options) + } - if (currentVersion === seederVersion) { - logger.info('Seeder already up to date, skipping...') - return - } + async runMigrationPostgres (db, options) { + return this.runVersionChainPostgres(db, options) + } - await db.query('BEGIN') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - if (err.code === '23505' || - err.code === '23503') { - logger.warn(`Ignored PostgreSQL error: ${err.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } - } - } + async runSeederSQLite (dbName, options) { + return this.runVersionChainSQLite(dbName, options) + } - await this.updateSeederVersion(db, seederVersion, 'postgres') - await db.query('COMMIT') - logger.info('Seeding completed successfully.') - } catch (err) { - await db.query('ROLLBACK') - logger.error('Seeding failed:', err) - throw err - } + async runSeederMySQL (db, options) { + return this.runVersionChainMySQL(db, options) + } + + async runSeederPostgres (db, options) { + return this.runVersionChainPostgres(db, options) } } +DatabaseProvider.SCHEMA_VERSIONS = SCHEMA_VERSIONS +DatabaseProvider.BASELINE_VERSION = BASELINE_VERSION + module.exports = DatabaseProvider diff --git a/src/data/providers/sqlite.js b/src/data/providers/sqlite.js index 53693e1c..20e80232 100644 --- a/src/data/providers/sqlite.js +++ b/src/data/providers/sqlite.js @@ -17,7 +17,9 @@ class SqliteDatabaseProvider extends DatabaseProvider { if (!sqliteConfig.databaseName.endsWith('.sqlite')) { sqliteConfig.databaseName += '.sqlite' } - const storageFolder = path.resolve(__dirname, '../sqlite_files') + const storageFolder = process.env.SQLITE_STORAGE_DIR + ? path.resolve(process.env.SQLITE_STORAGE_DIR) + : path.resolve(__dirname, '../sqlite_files') sqliteConfig.storage = path.resolve(storageFolder, sqliteConfig.databaseName) if (!fs.existsSync(storageFolder)) { fs.mkdirSync(storageFolder) diff --git a/src/data/seeders/mysql/db_seeder_mysql_baseline_v3.9.0.sql b/src/data/seeders/mysql/db_seeder_mysql_baseline_v3.9.0.sql new file mode 100644 index 00000000..93ad07a0 --- /dev/null +++ b/src/data/seeders/mysql/db_seeder_mysql_baseline_v3.9.0.sql @@ -0,0 +1,69 @@ +START TRANSACTION; + +INSERT INTO `Registries` (url, is_public, user_name, password, user_email, type, ca, insecure) +VALUES + ('docker.io', true, '', '', '', 'oci', NULL, false), + ('from_cache', true, '', '', '', 'oci', NULL, false), + ('https://huggingface.co', true, '', '', '', 'hf', NULL, false); + +INSERT INTO `CatalogItems` (name, description, category, publisher, disk_required, ram_required, picture, config_example, is_public, registry_id) +VALUES + ('Router', 'The built-in router for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('Debug', 'The built-in debugger for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('NATs', 'NATs server microservice for Edgelet', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, true, 1); + +INSERT INTO `Architectures` (id, name, image, description, network_catalog_item_id) +VALUES + (0, 'auto', 'iointegrator0.png', 'Architecture will be selected on provision', 1), + (1, 'amd64', 'iointegrator1.png', 'Standard x86_64 Linux. Compatible with common Linux distributions such as Ubuntu, Red Hat, and CentOS.', 1), + (2, 'arm64', 'iointegrator2.png', '64-bit ARM Linux (aarch64). Microservices for this architecture are built for ARM64 systems.', 1), + (3, 'riscv64', 'iointegrator3.png', 'RISC-V 64-bit Linux. Microservices for this architecture are built for RISC-V systems.', 1), + (4, 'arm', 'iointegrator4.png', '32-bit ARM Linux. Microservices for this architecture are built for 32-bit ARM systems.', 1); + +UPDATE `Fogs` +SET arch_id = 0 +WHERE arch_id IS NULL; + +INSERT INTO `CatalogItemImages` (catalog_item_id, arch_id, container_image) +VALUES + (1, 1, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 2, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 3, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 4, 'ghcr.io/eclipse-iofog/router:latest'), + (2, 1, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 2, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 3, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 4, 'ghcr.io/eclipse-iofog/debugger:latest'), + (3, 1, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 2, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 3, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 4, 'ghcr.io/eclipse-iofog/nats:latest'); + +INSERT IGNORE INTO AuthPolicy ( + id, + min_password_length, + require_uppercase, + require_lowercase, + require_digit, + password_max_age_days, + password_history_count, + max_failed_attempts, + lockout_duration_minutes, + access_token_ttl_seconds, + refresh_token_ttl_seconds, + refresh_rotation, + max_concurrent_sessions +) +VALUES (1, 12, true, true, true, 0, 5, 5, 15, 900, 3600, true, NULL); + +INSERT IGNORE INTO AuthGroups (name, is_system, mfa_required) +VALUES + ('admin', true, false), + ('sre', true, false), + ('developer', true, false), + ('viewer', true, false); + +INSERT IGNORE INTO AuthBootstrapMeta (id, completed_at) +VALUES (1, NULL); + +COMMIT; diff --git a/src/data/seeders/postgres/db_seeder_pg_baseline_v3.9.0.sql b/src/data/seeders/postgres/db_seeder_pg_baseline_v3.9.0.sql new file mode 100644 index 00000000..a8c5cf9a --- /dev/null +++ b/src/data/seeders/postgres/db_seeder_pg_baseline_v3.9.0.sql @@ -0,0 +1,73 @@ +START TRANSACTION; + +INSERT INTO "Registries" (url, is_public, user_name, password, user_email, type, ca, insecure) +VALUES + ('docker.io', true, '', '', '', 'oci', NULL, false), + ('from_cache', true, '', '', '', 'oci', NULL, false), + ('https://huggingface.co', true, '', '', '', 'hf', NULL, false); + +INSERT INTO "CatalogItems" (name, description, category, publisher, disk_required, ram_required, picture, config_example, is_public, registry_id) +VALUES + ('Router', 'The built-in router for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('Debug', 'The built-in debugger for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('NATs', 'NATs server microservice for Edgelet', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, true, 1); + +INSERT INTO "Architectures" (id, name, image, description, network_catalog_item_id) +VALUES + (0, 'auto', 'iointegrator0.png', 'Architecture will be selected on provision', 1), + (1, 'amd64', 'iointegrator1.png', 'Standard x86_64 Linux. Compatible with common Linux distributions such as Ubuntu, Red Hat, and CentOS.', 1), + (2, 'arm64', 'iointegrator2.png', '64-bit ARM Linux (aarch64). Microservices for this architecture are built for ARM64 systems.', 1), + (3, 'riscv64', 'iointegrator3.png', 'RISC-V 64-bit Linux. Microservices for this architecture are built for RISC-V systems.', 1), + (4, 'arm', 'iointegrator4.png', '32-bit ARM Linux. Microservices for this architecture are built for 32-bit ARM systems.', 1); + +UPDATE "Fogs" +SET arch_id = 0 +WHERE arch_id IS NULL; + +INSERT INTO "CatalogItemImages" (catalog_item_id, arch_id, container_image) +VALUES + (1, 1, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 2, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 3, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 4, 'ghcr.io/eclipse-iofog/router:latest'), + (2, 1, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 2, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 3, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 4, 'ghcr.io/eclipse-iofog/debugger:latest'), + (3, 1, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 2, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 3, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 4, 'ghcr.io/eclipse-iofog/nats:latest'); + +INSERT INTO "AuthPolicy" ( + id, + min_password_length, + require_uppercase, + require_lowercase, + require_digit, + password_max_age_days, + password_history_count, + max_failed_attempts, + lockout_duration_minutes, + access_token_ttl_seconds, + refresh_token_ttl_seconds, + refresh_rotation, + max_concurrent_sessions +) +VALUES (1, 12, true, true, true, 0, 5, 5, 15, 900, 3600, true, NULL) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO "AuthGroups" (name, is_system, mfa_required) +VALUES + ('admin', true, false), + ('sre', true, false), + ('developer', true, false), + ('viewer', true, false) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO "AuthBootstrapMeta" (id, completed_at) +OVERRIDING SYSTEM VALUE +VALUES (1, NULL) +ON CONFLICT (id) DO NOTHING; + +COMMIT; diff --git a/src/data/seeders/sqlite/db_seeder_sqlite_baseline_v3.9.0.sql b/src/data/seeders/sqlite/db_seeder_sqlite_baseline_v3.9.0.sql new file mode 100644 index 00000000..01cae2f3 --- /dev/null +++ b/src/data/seeders/sqlite/db_seeder_sqlite_baseline_v3.9.0.sql @@ -0,0 +1,65 @@ +INSERT INTO `Registries` (url, is_public, user_name, password, user_email, type, ca, insecure) +VALUES + ('docker.io', true, '', '', '', 'oci', NULL, false), + ('from_cache', true, '', '', '', 'oci', NULL, false), + ('https://huggingface.co', true, '', '', '', 'hf', NULL, false); + +INSERT INTO `CatalogItems` (name, description, category, publisher, disk_required, ram_required, picture, config_example, is_public, registry_id) +VALUES + ('Router', 'The built-in router for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('Debug', 'The built-in debugger for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('NATs', 'NATs server microservice for Edgelet', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, true, 1); + +INSERT INTO `Architectures` (id, name, image, description, network_catalog_item_id) +VALUES + (0, 'auto', 'iointegrator0.png', 'Architecture will be selected on provision', 1), + (1, 'amd64', 'iointegrator1.png', 'Standard x86_64 Linux. Compatible with common Linux distributions such as Ubuntu, Red Hat, and CentOS.', 1), + (2, 'arm64', 'iointegrator2.png', '64-bit ARM Linux (aarch64). Microservices for this architecture are built for ARM64 systems.', 1), + (3, 'riscv64', 'iointegrator3.png', 'RISC-V 64-bit Linux. Microservices for this architecture are built for RISC-V systems.', 1), + (4, 'arm', 'iointegrator4.png', '32-bit ARM Linux. Microservices for this architecture are built for 32-bit ARM systems.', 1); + +UPDATE `Fogs` +SET arch_id = 0 +WHERE arch_id IS NULL; + +INSERT INTO `CatalogItemImages` (catalog_item_id, arch_id, container_image) +VALUES + (1, 1, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 2, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 3, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 4, 'ghcr.io/eclipse-iofog/router:latest'), + (2, 1, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 2, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 3, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 4, 'ghcr.io/eclipse-iofog/debugger:latest'), + (3, 1, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 2, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 3, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 4, 'ghcr.io/eclipse-iofog/nats:latest'); + +INSERT OR IGNORE INTO AuthPolicy ( + id, + min_password_length, + require_uppercase, + require_lowercase, + require_digit, + password_max_age_days, + password_history_count, + max_failed_attempts, + lockout_duration_minutes, + access_token_ttl_seconds, + refresh_token_ttl_seconds, + refresh_rotation, + max_concurrent_sessions +) +VALUES (1, 12, true, true, true, 0, 5, 5, 15, 900, 3600, true, NULL); + +INSERT OR IGNORE INTO AuthGroups (name, is_system, mfa_required) +VALUES + ('admin', true, false), + ('sre', true, false), + ('developer', true, false), + ('viewer', true, false); + +INSERT OR IGNORE INTO AuthBootstrapMeta (id, completed_at) +VALUES (1, NULL); diff --git a/test/src/data/schema-version-chain-sqlite.test.js b/test/src/data/schema-version-chain-sqlite.test.js new file mode 100644 index 00000000..608fa660 --- /dev/null +++ b/test/src/data/schema-version-chain-sqlite.test.js @@ -0,0 +1,382 @@ +'use strict' + +const { expect } = require('chai') +const fs = require('fs') +const os = require('os') +const path = require('path') +const sqlite3 = require('sqlite3').verbose() + +const DatabaseProvider = require('../../../src/data/providers/database-provider') + +function sqliteGet (db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { + if (err) reject(err) + else resolve(row) + }) + }) +} + +function sqliteAll (db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) reject(err) + else resolve(rows) + }) + }) +} + +function sqliteClose (db) { + return new Promise((resolve, reject) => { + db.close((err) => { + if (err) reject(err) + else resolve() + }) + }) +} + +function openSqlite (dbPath) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(dbPath, (err) => { + if (err) reject(err) + else resolve(db) + }) + }) +} + +function cleanupSqlite (dbPath) { + for (const suffix of ['', '-wal', '-shm']) { + try { + fs.unlinkSync(dbPath + suffix) + } catch (_) { /* ignore */ } + } +} + +function normalizeDefault (value) { + if (value == null) { + return null + } + const text = String(value).trim().replace(/^['"]|['"]$/g, '').toLowerCase() + if (text === 'false' || text === '0') { + return '0' + } + if (text === 'true' || text === '1') { + return '1' + } + return text +} + +function normalizeIndexSql (sql) { + if (!sql) { + return '' + } + return sql + .replace(/\s+/g, ' ') + .replace(/IF NOT EXISTS /ig, '') + .replace(/"/g, '') + .trim() + .toLowerCase() +} + +async function dumpNormalizedSchema (db) { + const tables = await sqliteAll(db, ` + SELECT name FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'SchemaVersion' + ORDER BY name + `) + + const tableDump = [] + for (const { name } of tables) { + const cols = await sqliteAll(db, `PRAGMA table_info("${name}")`) + const fks = await sqliteAll(db, `PRAGMA foreign_key_list("${name}")`) + tableDump.push({ + table: name, + columns: cols + .map((col) => ({ + name: col.name, + type: String(col.type || '').toUpperCase(), + notnull: col.notnull, + dflt_value: normalizeDefault(col.dflt_value), + pk: col.pk + })) + .sort((left, right) => left.name.localeCompare(right.name)), + foreignKeys: fks + .map((fk) => ({ + table: fk.table, + from: fk.from, + to: fk.to, + on_delete: String(fk.on_delete || '').toUpperCase() + })) + .sort((left, right) => { + const key = (row) => `${row.table}|${row.from}|${row.to}|${row.on_delete}` + return key(left).localeCompare(key(right)) + }) + }) + } + + const indexes = await sqliteAll(db, ` + SELECT name, tbl_name, sql FROM sqlite_master + WHERE type = 'index' + AND name NOT LIKE 'sqlite_%' + AND tbl_name != 'SchemaVersion' + ORDER BY name + `) + + return JSON.stringify({ + tables: tableDump, + indexes: indexes.map((index) => ({ + name: index.name, + table: index.tbl_name, + sql: normalizeIndexSql(index.sql) + })) + }, null, 2) +} + +describe('schema version chain (sqlite)', function () { + this.timeout(60000) + + let dbPath + const provider = new DatabaseProvider() + + beforeEach(() => { + dbPath = path.join(os.tmpdir(), `controller-schema-chain-${Date.now()}-${Math.random()}.sqlite`) + }) + + afterEach(() => { + cleanupSqlite(dbPath) + }) + + it('fresh install reaches 3.9.0 with Hub registry and system catalog only', async () => { + await provider.runVersionChainSQLite(dbPath) + + const db = await openSqlite(dbPath) + try { + const versions = await sqliteAll(db, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id') + expect(versions).to.have.length(1) + expect(versions[0].migration_version).to.equal('3.9.0') + expect(versions[0].seeder_version).to.equal('3.9.0') + + const authGroups = await sqliteAll(db, 'SELECT name FROM AuthGroups ORDER BY name') + expect(authGroups.map((row) => row.name)).to.deep.equal(['admin', 'developer', 'sre', 'viewer']) + const authPolicy = await sqliteGet(db, 'SELECT id FROM AuthPolicy WHERE id = 1') + expect(authPolicy).to.not.equal(undefined) + const authBootstrap = await sqliteGet(db, 'SELECT id FROM AuthBootstrapMeta WHERE id = 1') + expect(authBootstrap).to.not.equal(undefined) + + const hub = await sqliteGet(db, "SELECT id, url, type FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf'") + expect(hub).to.not.equal(undefined) + expect(hub.id).to.equal(3) + expect(hub.type).to.equal('hf') + + const catalog = await sqliteAll(db, "SELECT name FROM CatalogItems WHERE category = 'SYSTEM' ORDER BY name") + expect(catalog.map((row) => row.name)).to.deep.equal(['Debug', 'NATs', 'Router']) + + const tables = await sqliteAll(db, "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('Models', 'RuntimeClasses', 'FogModels', 'FogRuntimeClasses', 'MicroserviceTemplates', 'MicroserviceTemplateVariables', 'MicroserviceEntrypoints', 'MicroserviceDevices', 'MicroserviceTmpfs', 'MicroserviceUlimits', 'MicroserviceModels', 'MicroserviceModelItems', 'HWInfos', 'USBInfos')") + const tableNames = tables.map((row) => row.name).sort() + expect(tableNames).to.deep.equal([ + 'FogModels', + 'FogRuntimeClasses', + 'MicroserviceDevices', + 'MicroserviceEntrypoints', + 'MicroserviceModelItems', + 'MicroserviceModels', + 'MicroserviceTemplateVariables', + 'MicroserviceTemplates', + 'MicroserviceTmpfs', + 'MicroserviceUlimits', + 'Models', + 'RuntimeClasses' + ]) + + const templateCols = await sqliteAll(db, 'PRAGMA table_info(MicroserviceTemplates)') + expect(templateCols.map((col) => col.name)).to.not.include('variables_json') + expect(templateCols.map((col) => col.name)).to.include('microservice_json') + + const archInfo = await sqliteGet(db, 'PRAGMA table_info(Architectures)') + const archCols = await sqliteAll(db, 'PRAGMA table_info(Architectures)') + expect(archInfo).to.not.equal(undefined) + expect(archCols.map((col) => col.name)).to.not.include('hal_catalog_item_id') + expect(archCols.map((col) => col.name)).to.not.include('bluetooth_catalog_item_id') + expect(archCols.map((col) => col.name)).to.include('network_catalog_item_id') + + const runtimeClasses = await sqliteAll(db, 'SELECT name FROM RuntimeClasses') + expect(runtimeClasses).to.deep.equal([]) + } finally { + await sqliteClose(db) + } + }) + + it('loads sequelize models against the migrated schema', async () => { + await provider.runVersionChainSQLite(dbPath) + + const Sequelize = require('sequelize') + const sequelize = new Sequelize({ + dialect: 'sqlite', + storage: dbPath, + logging: false + }) + const modelsDir = path.join(__dirname, '../../../src/data/models') + const db = {} + fs.readdirSync(modelsDir) + .filter((file) => file.endsWith('.js') && file !== 'index.js') + .forEach((file) => { + const model = require(path.join(modelsDir, file))(sequelize, Sequelize.DataTypes) + db[model.name] = model + }) + Object.keys(db).forEach((name) => { + if (db[name].associate) { + db[name].associate(db) + } + }) + + try { + const architectures = await db.Architecture.findAll({ order: [['id', 'ASC']] }) + expect(architectures).to.have.length(5) + expect(architectures[0].networkCatalogItemId).to.not.equal(undefined) + expect(architectures[0].halCatalogItemId).to.equal(undefined) + + const hub = await db.Registry.findOne({ where: { url: 'https://huggingface.co', type: 'hf' } }) + expect(hub).to.not.equal(null) + expect(hub.type).to.equal('hf') + expect(hub.insecure).to.equal(false) + + const fleetModels = await db.FleetModel.findAll() + expect(fleetModels).to.have.length(0) + const runtimeClasses = await db.RuntimeClass.findAll() + expect(runtimeClasses).to.have.length(0) + const microserviceTemplates = await db.MicroserviceTemplate.findAll() + expect(microserviceTemplates).to.have.length(0) + const microserviceTemplateVariables = await db.MicroserviceTemplateVariable.findAll() + expect(microserviceTemplateVariables).to.have.length(0) + + const fogs = await db.Fog.findAll() + expect(fogs).to.have.length(0) + } finally { + await sequelize.close() + } + }) + + it('upgrade from 3.8.0 applies 3.9.0 without dropping architecture rows', async () => { + await provider.runVersionChainSQLite(dbPath, { untilVersion: '3.8.0' }) + + const before = await openSqlite(dbPath) + let architectureNames + try { + const schema = await sqliteGet(before, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id DESC LIMIT 1') + expect(schema.migration_version).to.equal('3.8.0') + expect(schema.seeder_version).to.equal('3.8.0') + + const catalog = await sqliteAll(before, "SELECT name FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue') ORDER BY name") + expect(catalog.map((row) => row.name)).to.deep.equal(['HAL', 'RESTBlue']) + + const hub = await sqliteGet(before, "SELECT id FROM Registries WHERE url = 'https://huggingface.co'") + expect(hub).to.equal(undefined) + + const architectures = await sqliteAll(before, 'SELECT id, name FROM Architectures ORDER BY id') + expect(architectures).to.have.length(5) + architectureNames = architectures.map((row) => row.name) + } finally { + await sqliteClose(before) + } + + await provider.runVersionChainSQLite(dbPath) + + const after = await openSqlite(dbPath) + try { + const versions = await sqliteAll(after, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id') + expect(versions).to.have.length(2) + expect(versions[0].migration_version).to.equal('3.8.0') + expect(versions[0].seeder_version).to.equal('3.8.0') + expect(versions[1].migration_version).to.equal('3.9.0') + expect(versions[1].seeder_version).to.equal('3.9.0') + + const architectures = await sqliteAll(after, 'SELECT id, name FROM Architectures ORDER BY id') + expect(architectures).to.have.length(5) + expect(architectures.map((row) => row.name)).to.deep.equal(architectureNames) + + const catalog = await sqliteAll(after, "SELECT name FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')") + expect(catalog).to.deep.equal([]) + + const hub = await sqliteGet(after, "SELECT id, url, type FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf'") + expect(hub).to.not.equal(undefined) + expect(hub.type).to.equal('hf') + expect(hub.id).to.not.equal(1) + expect(hub.id).to.not.equal(2) + + const fogCols = await sqliteAll(after, 'PRAGMA table_info(Fogs)') + const fogColNames = fogCols.map((col) => col.name) + expect(fogColNames).to.not.include('device_scan_frequency') + expect(fogColNames).to.not.include('bluetooth') + expect(fogColNames).to.not.include('hal') + expect(fogColNames).to.include('model_status') + expect(fogColNames).to.include('runtime_classes') + expect(fogColNames).to.include('available_cdi_devices') + + const msCols = await sqliteAll(after, 'PRAGMA table_info(Microservices)') + const msColNames = msCols.map((col) => col.name) + expect(msColNames).to.include('sysctls') + expect(msColNames).to.include('run_as_group') + expect(msColNames).to.not.include('entrypoint') + expect(msColNames).to.not.include('commands') + expect(msColNames).to.not.include('ulimits') + expect(msColNames).to.not.include('devices') + expect(msColNames).to.not.include('tmpfs') + expect(msColNames).to.not.include('models') + + const tables = await sqliteAll(after, "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name") + const tableNames = tables.map((row) => row.name) + expect(tableNames).to.include.members([ + 'MicroserviceEntrypoints', + 'MicroserviceDevices', + 'MicroserviceTmpfs', + 'MicroserviceUlimits', + 'MicroserviceModels', + 'MicroserviceModelItems' + ]) + } finally { + await sqliteClose(after) + } + }) + + it('fresh baseline schema matches upgrade from 3.8.0', async () => { + const freshPath = dbPath + const upgradePath = dbPath + '.upgrade' + cleanupSqlite(upgradePath) + try { + await provider.runVersionChainSQLite(freshPath) + await provider.runVersionChainSQLite(upgradePath, { untilVersion: '3.8.0' }) + await provider.runVersionChainSQLite(upgradePath) + + const freshDb = await openSqlite(freshPath) + const upgradeDb = await openSqlite(upgradePath) + try { + const freshDump = await dumpNormalizedSchema(freshDb) + const upgradeDump = await dumpNormalizedSchema(upgradeDb) + expect(freshDump).to.equal(upgradeDump) + } finally { + await sqliteClose(freshDb) + await sqliteClose(upgradeDb) + } + } finally { + cleanupSqlite(upgradePath) + } + }) + + it('is idempotent when the version chain runs twice on a baseline database', async () => { + await provider.runVersionChainSQLite(dbPath) + await provider.runVersionChainSQLite(dbPath) + + const db = await openSqlite(dbPath) + try { + const versions = await sqliteAll(db, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id') + expect(versions).to.have.length(1) + expect(versions[0].migration_version).to.equal('3.9.0') + expect(versions[0].seeder_version).to.equal('3.9.0') + + const catalog = await sqliteAll(db, "SELECT name FROM CatalogItems WHERE category = 'SYSTEM' ORDER BY name") + expect(catalog.map((row) => row.name)).to.deep.equal(['Debug', 'NATs', 'Router']) + } finally { + await sqliteClose(db) + } + }) +}) diff --git a/test/src/integration/first-fog-reconcile-sqlite.test.js b/test/src/integration/first-fog-reconcile-sqlite.test.js index 96fba581..fe851bc6 100644 --- a/test/src/integration/first-fog-reconcile-sqlite.test.js +++ b/test/src/integration/first-fog-reconcile-sqlite.test.js @@ -57,9 +57,7 @@ describe('first-fog reconcile sqlite (R133)', function () { name: 'hub-edge', host: '127.0.0.1', archId: 1, - containerEngine: 'edgelet', - bluetoothEnabled: false, - abstractedHardwareEnabled: false + containerEngine: 'edgelet' } let fogUuid diff --git a/test/support/first-fog-sqlite-harness.js b/test/support/first-fog-sqlite-harness.js index 52c8df44..71e7f6b5 100644 --- a/test/support/first-fog-sqlite-harness.js +++ b/test/support/first-fog-sqlite-harness.js @@ -1,6 +1,7 @@ 'use strict' const fs = require('fs') +const os = require('os') const path = require('path') const express = require('express') @@ -15,7 +16,8 @@ const ENV_KEYS = [ 'OIDC_BOOTSTRAP_ADMIN_USERNAME', 'OIDC_BOOTSTRAP_ADMIN_PASSWORD', 'CONTROL_PLANE', - 'NODE_ENV' + 'NODE_ENV', + 'SQLITE_STORAGE_DIR' ] function snapshotEnv (keys) { @@ -38,8 +40,13 @@ function applyEnv (values) { } } +function sqliteStorageDir () { + return process.env.SQLITE_STORAGE_DIR || + path.resolve(__dirname, '../../src/data/sqlite_files') +} + function sqliteStoragePath (dbName) { - return path.resolve(__dirname, '../../src/data/sqlite_files', dbName) + return path.join(sqliteStorageDir(), dbName) } function cleanupSqliteFiles (dbName) { @@ -98,10 +105,13 @@ async function driveReconcileUntilReady (fogUuid, { async function createFirstFogSqliteHarness () { const envSnapshot = snapshotEnv(ENV_KEYS) const dbName = `first-fog-int-${Date.now()}-${Math.random().toString(36).slice(2)}.sqlite` + const storageDir = path.join(os.tmpdir(), 'controller-integration-sqlite') + fs.mkdirSync(storageDir, { recursive: true }) applyEnv({ DB_PROVIDER: 'sqlite', DB_NAME: dbName, + SQLITE_STORAGE_DIR: storageDir, AUTH_MODE: 'embedded', CONTROLLER_PUBLIC_URL: 'http://controller.test', AUTH_INSECURE_ALLOW_HTTP: 'true', From 6608d8eddac76f2aaa116ba82f84f8058af5049f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:50:29 +0300 Subject: [PATCH 03/55] Expose linked models and RuntimeClasses on the agent API. Agents pull only attached rows, ingest model/runtime status including podId, and drop hardware-inventory routes and device scan frequency from config. --- src/controllers/agent-controller.js | 24 +- src/routes/agent.js | 123 +++--- src/schemas/agent.js | 38 +- src/services/agent-service.js | 140 +++++-- src/services/change-tracking-service.js | 20 +- test/src/controllers/agent-controller.test.js | 85 +--- test/src/routes/agent-routes.test.js | 16 + test/src/services/agent-service.test.js | 388 ++++++++++++------ .../services/change-tracking-service.test.js | 11 + 9 files changed, 517 insertions(+), 328 deletions(-) create mode 100644 test/src/routes/agent-routes.test.js diff --git a/src/controllers/agent-controller.js b/src/controllers/agent-controller.js index 1c6419f9..f353ac19 100644 --- a/src/controllers/agent-controller.js +++ b/src/controllers/agent-controller.js @@ -52,6 +52,14 @@ const getAgentLinkedVolumeMountsEndpoint = async function (req, fog) { return { volumeMounts: await AgentService.getAgentLinkedVolumeMounts(fog) } } +const getAgentLinkedModelsEndpoint = async function (req, fog) { + return { models: await AgentService.getAgentLinkedModels(fog) } +} + +const getAgentLinkedRuntimeClassesEndpoint = async function (req, fog) { + return { runtimeClasses: await AgentService.getAgentLinkedRuntimeClasses(fog) } +} + const getAgentLogSessionsEndPoint = async function (req, fog) { return AgentService.getAgentLogSessions(fog) } @@ -78,18 +86,6 @@ const getAgentChangeVersionCommandEndPoint = async function (req, fog) { return AgentService.getAgentChangeVersionCommand(fog) } -const updateHalHardwareInfoEndPoint = async function (req, fog) { - const hardwareData = req.body - - return AgentService.updateHalHardwareInfo(hardwareData, fog) -} - -const updateHalUsbInfoEndPoint = async function (req, fog) { - const usbData = req.body - - return AgentService.updateHalUsbInfo(usbData, fog) -} - const deleteNodeEndPoint = async function (req, fog) { return AgentService.deleteNode(fog) } @@ -115,11 +111,11 @@ module.exports = { getAgentRegistriesEndPoint: AuthDecorator.checkFogToken(getAgentRegistriesEndPoint), getAgentTunnelEndPoint: AuthDecorator.checkFogToken(getAgentTunnelEndPoint), getAgentChangeVersionCommandEndPoint: AuthDecorator.checkFogToken(getAgentChangeVersionCommandEndPoint), - updateHalHardwareInfoEndPoint: AuthDecorator.checkFogToken(updateHalHardwareInfoEndPoint), - updateHalUsbInfoEndPoint: AuthDecorator.checkFogToken(updateHalUsbInfoEndPoint), deleteNodeEndPoint: AuthDecorator.checkFogToken(deleteNodeEndPoint), resetAgentConfigChangesEndPoint: AuthDecorator.checkFogToken(resetAgentConfigChangesEndPoint), getAgentLinkedVolumeMountsEndpoint: AuthDecorator.checkFogToken(getAgentLinkedVolumeMountsEndpoint), + getAgentLinkedModelsEndpoint: AuthDecorator.checkFogToken(getAgentLinkedModelsEndpoint), + getAgentLinkedRuntimeClassesEndpoint: AuthDecorator.checkFogToken(getAgentLinkedRuntimeClassesEndpoint), getControllerCAEndPoint: AuthDecorator.checkFogToken(getControllerCAEndPoint), getAgentLogSessionsEndPoint: AuthDecorator.checkFogToken(getAgentLogSessionsEndPoint), getAgentExecSessionsEndPoint: AuthDecorator.checkFogToken(getAgentExecSessionsEndPoint), diff --git a/src/routes/agent.js b/src/routes/agent.js index e78340df..a64b680a 100644 --- a/src/routes/agent.js +++ b/src/routes/agent.js @@ -300,7 +300,7 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/microservices', + path: '/api/v3/agent/models', middleware: async (req, res) => { logger.apiReq(req) @@ -316,9 +316,9 @@ module.exports = [ } ] - const getAgentMicroservicesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroservicesEndPoint, + const getAgentLinkedModelsEndpoint = ResponseDecorator.handleErrors(AgentController.getAgentLinkedModelsEndpoint, successCode, errorCodes) - const responseObject = await getAgentMicroservicesEndPoint(req) + const responseObject = await getAgentLinkedModelsEndpoint(req) res .status(responseObject.code) @@ -329,16 +329,12 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/microservices/:microserviceUuid', + path: '/api/v3/agent/runtimeClasses', middleware: async (req, res) => { logger.apiReq(req) const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] - }, { code: constants.HTTP_CODE_UNAUTHORIZED, errors: [Errors.AgentAuthenticationError, Errors.AuthenticationError] @@ -349,9 +345,12 @@ module.exports = [ } ] - const getAgentMicroserviceEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroserviceEndPoint, - successCode, errorCodes) - const responseObject = await getAgentMicroserviceEndPoint(req) + const getAgentLinkedRuntimeClassesEndpoint = ResponseDecorator.handleErrors( + AgentController.getAgentLinkedRuntimeClassesEndpoint, + successCode, + errorCodes + ) + const responseObject = await getAgentLinkedRuntimeClassesEndpoint(req) res .status(responseObject.code) @@ -361,8 +360,8 @@ module.exports = [ } }, { - method: 'post', - path: '/api/v3/agent/controller/register', + method: 'get', + path: '/api/v3/agent/microservices', middleware: async (req, res) => { logger.apiReq(req) @@ -375,23 +374,12 @@ module.exports = [ { code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, errors: [Errors.ServiceUnavailableError] - }, - { - code: constants.HTTP_CODE_FORBIDDEN, - errors: [Errors.ForbiddenError] - }, - { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] } ] - const registerControllerMicroserviceEndPoint = ResponseDecorator.handleErrors( - AgentController.registerControllerMicroserviceEndPoint, - successCode, - errorCodes - ) - const responseObject = await registerControllerMicroserviceEndPoint(req) + const getAgentMicroservicesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroservicesEndPoint, + successCode, errorCodes) + const responseObject = await getAgentMicroservicesEndPoint(req) res .status(responseObject.code) @@ -402,12 +390,16 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/registries', + path: '/api/v3/agent/microservices/:microserviceUuid', middleware: async (req, res) => { logger.apiReq(req) const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ + { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] + }, { code: constants.HTTP_CODE_UNAUTHORIZED, errors: [Errors.AgentAuthenticationError, Errors.AuthenticationError] @@ -418,9 +410,9 @@ module.exports = [ } ] - const getAgentRegistriesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentRegistriesEndPoint, + const getAgentMicroserviceEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroserviceEndPoint, successCode, errorCodes) - const responseObject = await getAgentRegistriesEndPoint(req) + const responseObject = await getAgentMicroserviceEndPoint(req) res .status(responseObject.code) @@ -430,17 +422,13 @@ module.exports = [ } }, { - method: 'get', - path: '/api/v3/agent/tunnel', + method: 'post', + path: '/api/v3/agent/controller/register', middleware: async (req, res) => { logger.apiReq(req) const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] - }, { code: constants.HTTP_CODE_UNAUTHORIZED, errors: [Errors.AgentAuthenticationError, Errors.AuthenticationError] @@ -448,12 +436,23 @@ module.exports = [ { code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, errors: [Errors.ServiceUnavailableError] + }, + { + code: constants.HTTP_CODE_FORBIDDEN, + errors: [Errors.ForbiddenError] + }, + { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] } ] - const getAgentTunnelEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentTunnelEndPoint, - successCode, errorCodes) - const responseObject = await getAgentTunnelEndPoint(req) + const registerControllerMicroserviceEndPoint = ResponseDecorator.handleErrors( + AgentController.registerControllerMicroserviceEndPoint, + successCode, + errorCodes + ) + const responseObject = await registerControllerMicroserviceEndPoint(req) res .status(responseObject.code) @@ -464,7 +463,7 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/version', + path: '/api/v3/agent/registries', middleware: async (req, res) => { logger.apiReq(req) @@ -477,16 +476,12 @@ module.exports = [ { code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, errors: [Errors.ServiceUnavailableError] - }, - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] } ] - const getAgentChangeVersionCommandEndPoint = ResponseDecorator.handleErrors( - AgentController.getAgentChangeVersionCommandEndPoint, successCode, errorCodes) - const responseObject = await getAgentChangeVersionCommandEndPoint(req) + const getAgentRegistriesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentRegistriesEndPoint, + successCode, errorCodes) + const responseObject = await getAgentRegistriesEndPoint(req) res .status(responseObject.code) @@ -496,13 +491,17 @@ module.exports = [ } }, { - method: 'put', - path: '/api/v3/agent/hal/hw', + method: 'get', + path: '/api/v3/agent/tunnel', middleware: async (req, res) => { logger.apiReq(req) - const successCode = constants.HTTP_CODE_NO_CONTENT + const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ + { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] + }, { code: constants.HTTP_CODE_UNAUTHORIZED, errors: [Errors.AgentAuthenticationError, Errors.AuthenticationError] @@ -510,16 +509,12 @@ module.exports = [ { code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, errors: [Errors.ServiceUnavailableError] - }, - { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] } ] - const updateHalHardwareInfoEndPoint = ResponseDecorator.handleErrors(AgentController.updateHalHardwareInfoEndPoint, + const getAgentTunnelEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentTunnelEndPoint, successCode, errorCodes) - const responseObject = await updateHalHardwareInfoEndPoint(req) + const responseObject = await getAgentTunnelEndPoint(req) res .status(responseObject.code) @@ -529,12 +524,12 @@ module.exports = [ } }, { - method: 'put', - path: '/api/v3/agent/hal/usb', + method: 'get', + path: '/api/v3/agent/version', middleware: async (req, res) => { logger.apiReq(req) - const successCode = constants.HTTP_CODE_NO_CONTENT + const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ { code: constants.HTTP_CODE_UNAUTHORIZED, @@ -545,14 +540,14 @@ module.exports = [ errors: [Errors.ServiceUnavailableError] }, { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] } ] - const updateHalUsbInfoEndPoint = ResponseDecorator.handleErrors(AgentController.updateHalUsbInfoEndPoint, - successCode, errorCodes) - const responseObject = await updateHalUsbInfoEndPoint(req) + const getAgentChangeVersionCommandEndPoint = ResponseDecorator.handleErrors( + AgentController.getAgentChangeVersionCommandEndPoint, successCode, errorCodes) + const responseObject = await getAgentChangeVersionCommandEndPoint(req) res .status(responseObject.code) diff --git a/src/schemas/agent.js b/src/schemas/agent.js index 4b5ffbcd..b2a37e9a 100644 --- a/src/schemas/agent.js +++ b/src/schemas/agent.js @@ -37,7 +37,6 @@ const updateAgentConfig = { logFileCount: { type: 'integer', minimum: 0 }, statusFrequency: { type: 'integer', minimum: 0 }, changeFrequency: { type: 'integer', minimum: 0 }, - deviceScanFrequency: { type: 'integer', minimum: 0 }, watchdogEnabled: { type: 'boolean' }, latitude: { type: 'number', minimum: -90, maximum: 90 }, longitude: { type: 'number', minimum: -180, maximum: 180 }, @@ -94,6 +93,17 @@ const updateAgentStatus = { type: 'array', items: { type: 'string' } }, + runtimeClasses: { + type: 'array', + items: { type: 'object' } + }, + availableCdiDevices: { + type: 'array', + items: { type: 'string' } + }, + modelStatus: { type: 'string' }, + activeModels: { type: 'integer', minimum: 0 }, + modelLastUpdate: { type: 'integer', minimum: 0 }, runtimeAgentPhase: { type: 'string' }, controlPlaneQuiesced: { type: 'boolean' }, gpsMode: { type: 'string' }, @@ -123,34 +133,14 @@ const microserviceStatus = { memoryUsage: { type: 'number' }, ipAddress: { type: 'string' }, ipAddressExternal: { type: 'string' }, - execSessionIds: { type: 'array', items: { type: 'string' } } + execSessionIds: { type: 'array', items: { type: 'string' } }, + podId: { type: 'string' } }, required: ['id'], additionalProperties: true } -const updateHardwareInfo = { - id: '/updateHardwareInfo', - type: 'object', - properties: { - info: { type: 'string' } - }, - required: ['info'], - additionalProperties: true -} - -const updateUsbInfo = { - id: '/updateUsbInfo', - type: 'object', - properties: { - info: { type: 'string' } - }, - required: ['info'], - additionalProperties: true -} - module.exports = { - mainSchemas: [agentProvision, agentDeprovision, updateAgentConfig, updateAgentGps, updateAgentStatus, - updateHardwareInfo, updateUsbInfo], + mainSchemas: [agentProvision, agentDeprovision, updateAgentConfig, updateAgentGps, updateAgentStatus], innerSchemas: [microserviceStatus] } diff --git a/src/services/agent-service.js b/src/services/agent-service.js index ffd4f65e..1276440b 100644 --- a/src/services/agent-service.js +++ b/src/services/agent-service.js @@ -13,6 +13,7 @@ const ChangeTrackingService = require('./change-tracking-service') const FogVersionCommandManager = require('../data/managers/iofog-version-command-manager') const { refreshProvisionKeyForFog } = require('./iofog-service') const RegistryManager = require('../data/managers/registry-manager') +const { toAgentRegistry } = require('./registry-service') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const { microserviceState, microserviceExecState } = require('../enums/microservice-state') @@ -21,13 +22,12 @@ const Validator = require('../schemas') const Errors = require('../helpers/errors') const AppHelper = require('../helpers/app-helper') const ErrorMessages = require('../helpers/error-messages') -const HWInfoManager = require('../data/managers/hw-info-manager') -const USBInfoManager = require('../data/managers/usb-info-manager') const TunnelManager = require('../data/managers/tunnel-manager') const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceService = require('../services/microservices-service') const ApplicationManager = require('../data/managers/application-manager') const constants = require('../helpers/constants') +const CatalogContainer = require('../helpers/microservice-container-catalog') const SecretManager = require('../data/managers/secret-manager') const ConfigMapManager = require('../data/managers/config-map-manager') const MicroserviceLogStatusManager = require('../data/managers/microservice-log-status-manager') @@ -36,11 +36,36 @@ const FogLogStatusManager = require('../data/managers/fog-log-status-manager') const RbacRoleManager = require('../data/managers/rbac-role-manager') const CHANGE_TRACKING_DEFAULT = {} -const CHANGE_TRACKING_KEYS = ['config', 'version', 'reboot', 'deleteNode', 'microserviceList', 'microserviceConfig', 'registries', 'tunnel', 'prune', 'routerChanged', 'volumeMounts', 'execSessions', 'microserviceLogs', 'fogLogs'] +const CHANGE_TRACKING_KEYS = [ + 'config', + 'version', + 'reboot', + 'deleteNode', + 'microserviceList', + 'microserviceConfig', + 'registries', + 'tunnel', + 'prune', + 'routerChanged', + 'volumeMounts', + 'execSessions', + 'microserviceLogs', + 'fogLogs', + 'models', + 'runtimeClasses', + 'microserviceModels' +] for (const key of CHANGE_TRACKING_KEYS) { CHANGE_TRACKING_DEFAULT[key] = false } +function _stringifyStatusField (value) { + if (value == null) { + return undefined + } + return typeof value === 'string' ? value : JSON.stringify(value) +} + const agentProvision = async function (provisionData, transaction) { await Validator.validate(provisionData, Validator.schemas.agentProvision) @@ -153,7 +178,6 @@ const getAgentConfig = async function (fog, transaction) { edgeGuardFrequency: fogData.edgeGuardFrequency, statusFrequency: fogData.statusFrequency, changeFrequency: fogData.changeFrequency, - deviceScanFrequency: fogData.deviceScanFrequency, watchdogEnabled: fogData.watchdogEnabled, latitude: fogData.latitude, longitude: fogData.longitude, @@ -180,7 +204,6 @@ const updateAgentConfig = async function (updateData, fog, transaction) { logFileCount: updateData.logFileCount, statusFrequency: updateData.statusFrequency, changeFrequency: updateData.changeFrequency, - deviceScanFrequency: updateData.deviceScanFrequency, watchdogEnabled: updateData.watchdogEnabled, latitude: updateData.latitude, longitude: updateData.longitude, @@ -267,7 +290,12 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { isReadyToRollback: agentStatus.isReadyToRollback, activeVolumeMounts: agentStatus.activeVolumeMounts, volumeMountLastUpdate: agentStatus.volumeMountLastUpdate, - gpsStatus: agentStatus.gpsStatus + gpsStatus: agentStatus.gpsStatus, + runtimeClasses: _stringifyStatusField(agentStatus.runtimeClasses), + availableCdiDevices: _stringifyStatusField(agentStatus.availableCdiDevices), + modelStatus: _stringifyStatusField(agentStatus.modelStatus), + activeModels: agentStatus.activeModels, + modelLastUpdate: agentStatus.modelLastUpdate } fogStatus = AppHelper.deleteUndefinedFields(fogStatus) @@ -316,7 +344,8 @@ const _updateMicroserviceStatuses = async function (microserviceStatus, fog, tra percentage: status.percentage, errorMessage: status.errorMessage, ipAddress: status.ipAddress, - execSessionIds: status.execSessionIds + execSessionIds: status.execSessionIds, + podId: status.podId } microserviceStatus = AppHelper.deleteUndefinedFields(microserviceStatus) const microservice = await MicroserviceManager.findOne({ @@ -470,6 +499,15 @@ const getAgentMicroservices = async function (fog, transaction) { schedule: microservice.schedule } + CatalogContainer.applyAgentContainerFields(responseMicroservice, microservice, { + cmd, + entrypoint: CatalogContainer.tokensFromRows(microservice.entrypoint, 'entrypoint'), + devices: CatalogContainer.devicesFromRows(microservice.devices), + tmpfs: CatalogContainer.tmpfsFromRows(microservice.tmpfs), + ulimits: CatalogContainer.ulimitsFromRows(microservice.ulimits), + models: CatalogContainer.catalogFromRows(microservice.microserviceModel, microservice.modelItems) + }) + // Resolve service account with rules from relationship const serviceAccountData = await _resolveServiceAccountRules(microservice.serviceAccount, transaction) if (serviceAccountData) { @@ -514,7 +552,7 @@ const getAgentMicroservice = async function (microserviceUuid, fog, transaction) const getAgentRegistries = async function (fog, transaction) { const registries = await RegistryManager.findAll({}, transaction) return { - registries + registries: registries.map(toAgentRegistry) } } @@ -555,26 +593,6 @@ const getAgentChangeVersionCommand = async function (fog, transaction) { return response } -const updateHalHardwareInfo = async function (hardwareData, fog, transaction) { - await Validator.validate(hardwareData, Validator.schemas.updateHardwareInfo) - - hardwareData.iofogUuid = fog.uuid - - await HWInfoManager.updateOrCreate({ - iofogUuid: fog.uuid - }, hardwareData, transaction) -} - -const updateHalUsbInfo = async function (usbData, fog, transaction) { - await Validator.validate(usbData, Validator.schemas.updateUsbInfo) - - usbData.iofogUuid = fog.uuid - - await USBInfoManager.updateOrCreate({ - iofogUuid: fog.uuid - }, usbData, transaction) -} - const deleteNode = async function (fog, transaction) { await FogManager.delete({ uuid: fog.uuid @@ -775,6 +793,68 @@ const getAgentLinkedVolumeMounts = async function (fog, transaction) { return volumeMounts } +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +const getAgentLinkedModels = async function (fog, transaction) { + if (!fog || typeof fog.getModels !== 'function') { + return [] + } + + const resources = await fog.getModels({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction + }) + + const models = [] + for (const resource of resources) { + const row = _toPlain(resource) + if (!row || !row.uuid || !row.name) { + continue + } + models.push({ + uuid: row.uuid, + name: row.name, + repo: row.repo, + revision: row.revision != null ? row.revision : '', + registryId: row.registryId, + files: Array.isArray(row.files) ? row.files : [], + format: row.format + }) + } + return models +} + +const getAgentLinkedRuntimeClasses = async function (fog, transaction) { + if (!fog || typeof fog.getRuntimeClassLinks !== 'function') { + return [] + } + + const resources = await fog.getRuntimeClassLinks({ + attributes: ['name', 'handler'], + joinTableAttributes: [], + transaction + }) + + const runtimeClasses = [] + for (const resource of resources) { + const row = _toPlain(resource) + if (!row || !row.name || !row.handler) { + continue + } + runtimeClasses.push({ + name: row.name, + handler: row.handler + }) + } + return runtimeClasses +} + module.exports = { agentProvision: TransactionDecorator.generateTransaction(agentProvision), agentDeprovision: TransactionDecorator.generateTransaction(agentDeprovision), @@ -789,10 +869,10 @@ module.exports = { getAgentRegistries: TransactionDecorator.generateTransaction(getAgentRegistries), getAgentTunnel: TransactionDecorator.generateTransaction(getAgentTunnel), getAgentChangeVersionCommand: TransactionDecorator.generateTransaction(getAgentChangeVersionCommand), - updateHalHardwareInfo: TransactionDecorator.generateTransaction(updateHalHardwareInfo), - updateHalUsbInfo: TransactionDecorator.generateTransaction(updateHalUsbInfo), deleteNode: TransactionDecorator.generateTransaction(deleteNode), getAgentLinkedVolumeMounts: TransactionDecorator.generateTransaction(getAgentLinkedVolumeMounts), + getAgentLinkedModels: TransactionDecorator.generateTransaction(getAgentLinkedModels), + getAgentLinkedRuntimeClasses: TransactionDecorator.generateTransaction(getAgentLinkedRuntimeClasses), getControllerCA, getAgentLogSessions: TransactionDecorator.generateTransaction(getAgentLogSessions), getAgentExecSessions: TransactionDecorator.generateTransaction(getAgentExecSessions) diff --git a/src/services/change-tracking-service.js b/src/services/change-tracking-service.js index 3174fd74..7ab2eff0 100644 --- a/src/services/change-tracking-service.js +++ b/src/services/change-tracking-service.js @@ -16,15 +16,20 @@ const events = Object.freeze({ prune: false, routerChanged: false, volumeMounts: false, - execSessions: false + execSessions: false, + models: false, + runtimeClasses: false, + microserviceModels: false }, microserviceFull: { microserviceConfig: true, - microserviceList: true + microserviceList: true, + microserviceModels: true }, microserviceCommon: { microserviceConfig: true, - microserviceList: true + microserviceList: true, + microserviceModels: true }, microserviceList: { microserviceList: true @@ -67,6 +72,15 @@ const events = Object.freeze({ }, fogLogs: { fogLogs: true + }, + models: { + models: true + }, + runtimeClasses: { + runtimeClasses: true + }, + microserviceModels: { + microserviceModels: true } }) diff --git a/test/src/controllers/agent-controller.test.js b/test/src/controllers/agent-controller.test.js index 17082bb3..7cdec596 100644 --- a/test/src/controllers/agent-controller.test.js +++ b/test/src/controllers/agent-controller.test.js @@ -139,7 +139,6 @@ describe('Agent Controller', () => { def('logFileCount', () => 5) def('statusFrequency', () => 60) def('changeFrequency', () => 30) - def('deviceScanFrequency', () => 40) def('watchdogEnabled', () => true) def('latitude', () => 30) def('longitude', () => 40) @@ -158,7 +157,6 @@ describe('Agent Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - devicesScanFrequency: $deviceScanFrequency, watchdogEnabled: $watchdogEnabled, latitude: $latitude, longitude: $longitude, @@ -187,7 +185,6 @@ describe('Agent Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - devicesScanFrequency: $deviceScanFrequency, watchdogEnabled: $watchdogEnabled, latitude: $latitude, longitude: $longitude, @@ -535,85 +532,35 @@ describe('Agent Controller', () => { }) }) - describe('updateHalHardwareInfoEndPoint()', () => { + describe('getAgentLinkedModelsEndpoint()', () => { def('fog', () => 'fog!') - - def('info', () => 'testInfo') - - def('req', () => ({ - body: { - info: $info, - }, - })) - def('response', () => Promise.resolve()) - def('subject', () => $subject.updateHalHardwareInfoEndPoint($req, $fog)) + def('req', () => ({ body: {} })) + def('response', () => Promise.resolve([])) + def('subject', () => $subject.getAgentLinkedModelsEndpoint($req, $fog)) beforeEach(() => { - $sandbox.stub(AgentService, 'updateHalHardwareInfo').returns($response) - }) - - it('calls AgentService.updateHalHardwareInfo with correct args', async () => { - await $subject - expect(AgentService.updateHalHardwareInfo).to.have.been.calledWith({ - info: $info, - }, $fog) + $sandbox.stub(AgentService, 'getAgentLinkedModels').returns($response) }) - context('when AgentService#updateHalHardwareInfo fails', () => { - const error = 'Error!' - - def('response', () => Promise.reject(error)) - - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when AgentService#updateHalHardwareInfo succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.equal(undefined) - }) + it('returns linked models for the requesting fog', async () => { + await expect($subject).to.eventually.eql({ models: [] }) + expect(AgentService.getAgentLinkedModels).to.have.been.calledWith($fog) }) }) - describe('updateHalUsbInfoEndPoint()', () => { + describe('getAgentLinkedRuntimeClassesEndpoint()', () => { def('fog', () => 'fog!') - - def('info', () => 'testInfo') - - def('req', () => ({ - body: { - info: $info, - }, - })) - def('response', () => Promise.resolve()) - def('subject', () => $subject.updateHalUsbInfoEndPoint($req, $fog)) + def('req', () => ({ body: {} })) + def('response', () => Promise.resolve([])) + def('subject', () => $subject.getAgentLinkedRuntimeClassesEndpoint($req, $fog)) beforeEach(() => { - $sandbox.stub(AgentService, 'updateHalUsbInfo').returns($response) - }) - - it('calls AgentService.updateHalUsbInfo with correct args', async () => { - await $subject - expect(AgentService.updateHalUsbInfo).to.have.been.calledWith({ - info: $info, - }, $fog) + $sandbox.stub(AgentService, 'getAgentLinkedRuntimeClasses').returns($response) }) - context('when AgentService#updateHalUsbInfo fails', () => { - const error = 'Error!' - - def('response', () => Promise.reject(error)) - - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when AgentService#updateHalUsbInfo succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.equal(undefined) - }) + it('returns linked runtime classes for the requesting fog', async () => { + await expect($subject).to.eventually.eql({ runtimeClasses: [] }) + expect(AgentService.getAgentLinkedRuntimeClasses).to.have.been.calledWith($fog) }) }) diff --git a/test/src/routes/agent-routes.test.js b/test/src/routes/agent-routes.test.js new file mode 100644 index 00000000..50e440b0 --- /dev/null +++ b/test/src/routes/agent-routes.test.js @@ -0,0 +1,16 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Agent routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/agent.js'), 'utf8') + + it('registers fog-token model and runtime class list routes', () => { + expect(source).to.include("path: '/api/v3/agent/models'") + expect(source).to.include("path: '/api/v3/agent/runtimeClasses'") + }) + + it('does not register agent hardware inventory routes', () => { + expect(source).to.not.include('/agent/hal/') + }) +}) diff --git a/test/src/services/agent-service.test.js b/test/src/services/agent-service.test.js index 8163d110..197a361f 100644 --- a/test/src/services/agent-service.test.js +++ b/test/src/services/agent-service.test.js @@ -18,8 +18,6 @@ const RegistryManager = require('../../../src/data/managers/registry-manager') const TunnelManager = require('../../../src/data/managers/tunnel-manager') const ioFogVersionCommandManager = require('../../../src/data/managers/iofog-version-command-manager') const ioFogProvisionKeyManager = require('../../../src/data/managers/iofog-provision-key-manager') -const HWInfoManager = require('../../../src/data/managers/hw-info-manager') -const USBInfoManager = require('../../../src/data/managers/usb-info-manager') const Sequelize = require('sequelize') const Op = Sequelize.Op const path = require('path') @@ -450,7 +448,6 @@ describe('Agent Service', () => { logFileCount: agentConfig.logFileCount, statusFrequency: agentConfig.statusFrequency, changeFrequency: agentConfig.changeFrequency, - deviceScanFrequency: agentConfig.deviceScanFrequency, watchdogEnabled: agentConfig.watchdogEnabled, latitude: agentConfig.latitude, longitude: agentConfig.longitude, @@ -486,6 +483,7 @@ describe('Agent Service', () => { expect(ioFogManager.update).to.have.been.calledWith({ uuid: $uuid, }, expectedFogUpdate, transaction) + expect(expectedFogUpdate).to.not.have.property('deviceScanFrequency') }) context('when ioFogManager#update fails', () => { @@ -507,21 +505,51 @@ describe('Agent Service', () => { }) }) - describe('.getAgentConfigChanges()', () => { - const configChanges = { - config: undefined, - version: undefined, - reboot: undefined, - deleteNode: undefined, - microserviceList: undefined, - microserviceConfig: undefined, - routing: undefined, - registries: undefined, - tunnel: undefined, - routerChanged: undefined, - prune: undefined, + describe('.getAgentConfig()', () => { + const transaction = {} + const fogData = { + networkInterface: 'eth0', + containerEngineUrl: 'unix:///run/edgelet/containerd.sock', + diskLimit: 50, + diskDirectory: '/var/lib/iofog/', + memoryLimit: 4096, + cpuLimit: 80, + logLimit: 10, + logDirectory: '/var/log/iofog/', + logFileCount: 10, + gpsMode: 'auto', + gpsDevice: '', + gpsScanFrequency: 60, + edgeGuardFrequency: 0, + statusFrequency: 10, + changeFrequency: 20, + watchdogEnabled: false, + latitude: 0, + longitude: 0, + logLevel: 'INFO', + availableDiskThreshold: 20, + pruningFrequency: 0, + timeZone: 'UTC', + deviceScanFrequency: 50, } + def('uuid', () => 'testUuid') + def('fog', () => ({ uuid: $uuid })) + def('subject', () => $subject.getAgentConfig($fog, transaction)) + + beforeEach(() => { + $sandbox.stub(ioFogManager, 'findOne').resolves(fogData) + }) + + it('omits deviceScanFrequency from the agent config payload', async () => { + const result = await $subject + expect(result).to.not.have.property('deviceScanFrequency') + expect(result.statusFrequency).to.equal(fogData.statusFrequency) + expect(result.edgeGuardFrequency).to.equal(fogData.edgeGuardFrequency) + }) + }) + + describe('.getAgentConfigChanges()', () => { const transaction = {} const error = 'Error!' @@ -535,7 +563,7 @@ describe('Agent Service', () => { def('subject', () => $subject.getAgentConfigChanges($fog, transaction)) - def('getByFogIdResponse', () => 'getByFogIdResponse') + def('getByFogIdResponse', () => []) def('updateIfChangedResponse', () => Promise.resolve()) beforeEach(() => { @@ -548,6 +576,13 @@ describe('Agent Service', () => { expect(ChangeTrackingService.getByIoFogUuid).to.have.been.calledWith($uuid, transaction) }) + it('includes additive change flags as false by default', async () => { + const result = await $subject + expect(result.models).to.equal(false) + expect(result.runtimeClasses).to.equal(false) + expect(result.microserviceModels).to.equal(false) + }) + context('when ChangeTrackingService#getByIoFogUuid() fails', () => { def('getByFogIdResponse', () => Promise.reject(error)) @@ -1349,11 +1384,37 @@ describe('Agent Service', () => { expect(msvc.application).to.equal('testApp') expect(msvc.registryId).to.equal(microserviceResponse.microservices[0].registryId) expect(msvc.cmd).to.deep.equal(microserviceResponse.microservices[0].cmd) + expect(msvc.commands).to.deep.equal(microserviceResponse.microservices[0].cmd) expect(msvc.extraHosts).to.deep.equal(microserviceResponse.microservices[0].extraHosts) expect(msvc.isController).to.equal(true) expect(msvc.isRouter).to.equal(false) expect(msvc.isNats).to.equal(false) }) + + it('emits catalog models and omits empty argv arrays', async () => { + MicroserviceManager.findAllActiveApplicationMicroservices.resolves([{ + ...microserviceWithValidImage, + cmd: [], + entrypoint: [], + microserviceModel: { bindPath: '/models', permissions: 'ro' }, + modelItems: [{ id: 1, name: 'test-model' }], + cpus: 1.5, + workingDir: '/app' + }]) + + const result = await $subject + const msvc = result.microservices[0] + expect(msvc).to.not.have.property('entrypoint') + expect(msvc).to.not.have.property('commands') + expect(msvc.cmd).to.eql([]) + expect(msvc.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(msvc.cpus).to.equal(1.5) + expect(msvc.workingDir).to.equal('/app') + }) }) }) }) @@ -1437,7 +1498,7 @@ describe('Agent Service', () => { def('subject', () => $subject.getAgentRegistries($fog, transaction)) - def('getAgentRegistriesResponse', () => Promise.resolve()) + def('getAgentRegistriesResponse', () => Promise.resolve([])) beforeEach(() => { $sandbox.stub(RegistryManager, 'findAll').returns($getAgentRegistriesResponse) @@ -1460,6 +1521,55 @@ describe('Agent Service', () => { it(`succeeds`, () => { return expect($subject).to.eventually.have.property('registries') }) + + context('with registry rows', () => { + def('getAgentRegistriesResponse', () => Promise.resolve([{ + id: 3, + url: 'https://huggingface.co', + isPublic: true, + username: '', + password: '', + userEmail: '', + type: 'hf', + ca: null, + insecure: false + }, { + id: 4, + url: 'https://registry.example.com', + isPublic: false, + username: 'user', + password: 'secret', + userEmail: 'user@example.com', + type: 'oci', + ca: 'Y2VydA==', + insecure: true + }])) + + it('includes type, ca, and insecure on every registry', async () => { + const result = await $subject + expect(result.registries).to.eql([{ + id: 3, + url: 'https://huggingface.co', + isPublic: true, + username: '', + password: '', + userEmail: '', + type: 'hf', + ca: '', + insecure: false + }, { + id: 4, + url: 'https://registry.example.com', + isPublic: false, + username: 'user', + password: 'secret', + userEmail: 'user@example.com', + type: 'oci', + ca: 'Y2VydA==', + insecure: true + }]) + }) + }) }) }) @@ -1612,139 +1722,169 @@ describe('Agent Service', () => { }) }) - describe('.updateHalHardwareInfo()', () => { + describe('.getAgentLinkedModels()', () => { const transaction = {} - const error = 'Error!' def('uuid', () => 'testUuid') - + def('getModels', () => $sandbox.stub().resolves([])) def('fog', () => ({ uuid: $uuid, + getModels: $getModels, })) - - def('info', () => 'testInfo') - def('hardwareData', () => ({ - info: $info, - })) - - def('response', () => ({ - versionCommand: $versionCommandLine, - provisionKey: $provisionKey, - expirationTime: $expirationTime, - })) - - def('subject', () => $subject.updateHalHardwareInfo($hardwareData, $fog, transaction)) - - def('validatorResponse', () => Promise.resolve(true)) - def('hwResponse', () => Promise.resolve()) - - beforeEach(() => { - $sandbox.stub(Validator, 'validate').returns($validatorResponse) - $sandbox.stub(HWInfoManager, 'updateOrCreate').returns($hwResponse) - }) - - it('calls Validator#validate() with correct args', async () => { - await $subject - expect(Validator.validate).to.have.been.calledWith($hardwareData, Validator.schemas.updateHardwareInfo) - }) - - context('when Validator#validate() fails', () => { - def('validatorResponse', () => Promise.reject(error)) - - it(`fails with ${error}`, () => { - return expect($subject).to.be.rejectedWith(error) + def('subject', () => $subject.getAgentLinkedModels($fog, transaction)) + + it('returns an empty list when the fog has no linked models', async () => { + await expect($subject).to.eventually.eql([]) + expect($getModels).to.have.been.calledWith({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction, }) }) - context('when Validator#validate() succeeds', () => { - it('calls HWInfoManager#updateOrCreate() with correct args', async () => { - await $subject - expect(HWInfoManager.updateOrCreate).to.have.been.calledWith({ - iofogUuid: $uuid, - }, $hardwareData, transaction) - - context('when HWInfoManager#updateOrCreate() fails', () => { - def('hwResponse', () => Promise.reject(error)) - - it(`fails with ${error}`, () => { - return expect($subject).to.be.equal(undefined) - }) - }) - - context('when HWInfoManager#updateOrCreate() succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.equal(undefined) - }) - }) - }) + it('maps linked model rows to the wire object', async () => { + $getModels.resolves([{ + uuid: '3f2c8a1e-2b64-4c0d-9f11-0a1b2c3d4e5f', + name: 'test-model', + repo: 'org/repo', + revision: null, + registryId: 3, + files: ['file.gguf'], + format: 'gguf', + toJSON () { + return { + uuid: this.uuid, + name: this.name, + repo: this.repo, + revision: this.revision, + registryId: this.registryId, + files: this.files, + format: this.format + } + } + }]) + + await expect($subject).to.eventually.eql([{ + uuid: '3f2c8a1e-2b64-4c0d-9f11-0a1b2c3d4e5f', + name: 'test-model', + repo: 'org/repo', + revision: '', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }]) }) }) - describe('.updateHalUsbInfo()', () => { + describe('.getAgentLinkedRuntimeClasses()', () => { const transaction = {} - const error = 'Error!' def('uuid', () => 'testUuid') - + def('getRuntimeClassLinks', () => $sandbox.stub().resolves([])) def('fog', () => ({ uuid: $uuid, + getRuntimeClassLinks: $getRuntimeClassLinks, })) + def('subject', () => $subject.getAgentLinkedRuntimeClasses($fog, transaction)) + + it('returns an empty list when the fog has no linked runtime classes', async () => { + await expect($subject).to.eventually.eql([]) + expect($getRuntimeClassLinks).to.have.been.calledWith({ + attributes: ['name', 'handler'], + joinTableAttributes: [], + transaction, + }) + }) - def('info', () => 'testInfo') - def('usbData', () => ({ - info: $info, - })) + it('maps linked runtime class rows to the wire object', async () => { + $getRuntimeClassLinks.resolves([{ + name: 'spin', + handler: 'spin', + toJSON () { + return { + name: this.name, + handler: this.handler + } + } + }]) - def('response', () => ({ - versionCommand: $versionCommandLine, - provisionKey: $provisionKey, - expirationTime: $expirationTime, - })) + await expect($subject).to.eventually.eql([{ + name: 'spin', + handler: 'spin' + }]) + }) + }) - def('subject', () => $subject.updateHalUsbInfo($usbData, $fog, transaction)) + describe('.updateAgentStatus() additive fields', () => { + const transaction = {} + const microserviceStatus = '[{"id":"testUuid","containerId":"ctr-1","status":"RUNNING","podId":"pod-1"}]' + const agentStatus = { + daemonStatus: 'RUNNING', + daemonOperatingDuration: 25, + daemonLastStart: 1, + warningMessage: '', + memoryUsage: 1, + diskUsage: 1, + cpuUsage: 1, + memoryViolation: false, + diskViolation: false, + cpuViolation: false, + systemAvailableDisk: 1, + systemAvailableMemory: 1, + systemTotalCpu: 1, + repositoryCount: 0, + repositoryStatus: '[]', + systemTime: 1, + lastStatusTime: 1, + ipAddress: '127.0.0.1', + ipAddressExternal: '127.0.0.1', + lastCommandTime: 1, + tunnelStatus: '{}', + version: '1.1.0', + isReadyToUpgrade: false, + isReadyToRollback: false, + gpsStatus: 'OK', + microserviceStatus, + runtimeClasses: [{ name: 'spin', handler: 'spin', source: 'managed' }], + availableCdiDevices: ['nvidia.com/gpu=0'], + modelStatus: '[]', + activeModels: 2, + modelLastUpdate: 1710000000, + } - def('validatorResponse', () => Promise.resolve(true)) - def('usbResponse', () => Promise.resolve()) + def('uuid', () => 'testUuid') + def('fog', () => ({ uuid: $uuid })) + def('subject', () => $subject.updateAgentStatus(agentStatus, $fog, transaction)) beforeEach(() => { - $sandbox.stub(Validator, 'validate').returns($validatorResponse) - $sandbox.stub(USBInfoManager, 'updateOrCreate').returns($usbResponse) + $sandbox.stub(Validator, 'validate').resolves(true) + $sandbox.spy(AppHelper, 'deleteUndefinedFields') + $sandbox.stub(ioFogManager, 'findOne').resolves({ warningMessage: '' }) + $sandbox.stub(FogPlatformStatusManager, 'getParsedStatus').resolves(null) + $sandbox.stub(ioFogManager, 'update').resolves() + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $uuid }) }) - it('calls Validator#validate() with correct args', async () => { + it('persists additive fog status fields and microservice podId', async () => { await $subject - expect(Validator.validate).to.have.been.calledWith($usbData, Validator.schemas.updateUsbInfo) - }) - - context('when Validator#validate() fails', () => { - def('validatorResponse', () => Promise.reject(error)) - - it(`fails with ${error}`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when Validator#validate() succeeds', () => { - it('calls USBInfoManager#updateOrCreate() with correct args', async () => { - await $subject - expect(USBInfoManager.updateOrCreate).to.have.been.calledWith({ - iofogUuid: $uuid, - }, $usbData, transaction) - - context('when USBInfoManager#updateOrCreate() fails', () => { - def('usbResponse', () => Promise.reject(error)) - - it(`fails with ${error}`, () => { - return expect($subject).to.be.equal(undefined) - }) - }) - - context('when USBInfoManager#updateOrCreate() succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.equal(undefined) - }) - }) - }) + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: $uuid }, + sinon.match({ + runtimeClasses: JSON.stringify(agentStatus.runtimeClasses), + availableCdiDevices: JSON.stringify(agentStatus.availableCdiDevices), + modelStatus: '[]', + activeModels: 2, + modelLastUpdate: 1710000000, + }), + transaction + ) + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: 'testUuid' }, + sinon.match({ podId: 'pod-1', containerId: 'ctr-1' }), + transaction + ) }) }) diff --git a/test/src/services/change-tracking-service.test.js b/test/src/services/change-tracking-service.test.js index 09d34082..a8d6459f 100644 --- a/test/src/services/change-tracking-service.test.js +++ b/test/src/services/change-tracking-service.test.js @@ -198,4 +198,15 @@ describe('ChangeTracking Service', () => { }) }) }) + + describe('events', () => { + it('includes additive model and runtime class flags', () => { + expect(ChangeTrackingService.events.clean.models).to.equal(false) + expect(ChangeTrackingService.events.clean.runtimeClasses).to.equal(false) + expect(ChangeTrackingService.events.clean.microserviceModels).to.equal(false) + expect(ChangeTrackingService.events.models).to.eql({ models: true }) + expect(ChangeTrackingService.events.runtimeClasses).to.eql({ runtimeClasses: true }) + expect(ChangeTrackingService.events.microserviceModels).to.eql({ microserviceModels: true }) + }) + }) }) From c0220b4440262656616034657eb3ed00a5a36b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:50:34 +0300 Subject: [PATCH 04/55] Add fleet model CRUD, YAML, and per-fog linking. Operators distribute models like volume mounts; unlink and delete refuse while a microservice still binds the name. --- src/controllers/model-controller.js | 63 +++ src/data/managers/fleet-model-manager.js | 33 ++ src/helpers/error-messages.js | 50 ++- src/routes/models.js | 187 ++++++++ src/schemas/model.js | 101 +++++ src/services/model-service.js | 271 +++++++++++ src/services/yaml-parser-service.js | 208 ++++++++- test/src/routes/models-routes.test.js | 19 + test/src/services/model-service.test.js | 327 ++++++++++++++ test/src/services/yaml-parser-service.test.js | 421 ++++++++++++++++++ 10 files changed, 1675 insertions(+), 5 deletions(-) create mode 100644 src/controllers/model-controller.js create mode 100644 src/data/managers/fleet-model-manager.js create mode 100644 src/routes/models.js create mode 100644 src/schemas/model.js create mode 100644 src/services/model-service.js create mode 100644 test/src/routes/models-routes.test.js create mode 100644 test/src/services/model-service.test.js diff --git a/src/controllers/model-controller.js b/src/controllers/model-controller.js new file mode 100644 index 00000000..c9206150 --- /dev/null +++ b/src/controllers/model-controller.js @@ -0,0 +1,63 @@ +const ModelService = require('../services/model-service') +const YAMLParserService = require('../services/yaml-parser-service') + +const listModelsEndpoint = async (req) => { + return ModelService.listModelsEndpoint() +} + +const getModelEndpoint = async (req) => { + return ModelService.getModelEndpoint(req.params.name) +} + +const createModelEndpoint = async (req) => { + return ModelService.createModelEndpoint(req.body) +} + +const updateModelEndpoint = async (req) => { + return ModelService.updateModelEndpoint(req.params.name, req.body) +} + +const deleteModelEndpoint = async (req) => { + return ModelService.deleteModelEndpoint(req.params.name) +} + +const createModelYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const modelData = await YAMLParserService.parseModelFile(fileContent) + return ModelService.createModelEndpoint(modelData) +} + +const upsertModelYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const name = req.params.name + const modelData = await YAMLParserService.parseModelFile(fileContent, { + isUpdate: true, + modelName: name + }) + return ModelService.upsertModelEndpoint(name, modelData) +} + +const getModelLinkEndpoint = async (req) => { + return ModelService.getModelLinkEndpoint(req.params.name) +} + +const linkModelEndpoint = async (req) => { + return ModelService.linkModelEndpoint(req.params.name, req.body.fogUuids) +} + +const unlinkModelEndpoint = async (req) => { + return ModelService.unlinkModelEndpoint(req.params.name, req.body.fogUuids) +} + +module.exports = { + listModelsEndpoint, + getModelEndpoint, + createModelEndpoint, + updateModelEndpoint, + deleteModelEndpoint, + createModelYamlEndpoint, + upsertModelYamlEndpoint, + getModelLinkEndpoint, + linkModelEndpoint, + unlinkModelEndpoint +} diff --git a/src/data/managers/fleet-model-manager.js b/src/data/managers/fleet-model-manager.js new file mode 100644 index 00000000..921c707f --- /dev/null +++ b/src/data/managers/fleet-model-manager.js @@ -0,0 +1,33 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const FleetModel = models.FleetModel + +const modelExcludedFields = [ + 'created_at', + 'updated_at' +] + +class FleetModelManager extends BaseManager { + getEntity () { + return FleetModel + } + + findOne (where, transaction) { + return FleetModel.findOne({ + where, + attributes: { exclude: modelExcludedFields }, + transaction + }) + } + + findAll (where, transaction) { + return FleetModel.findAll({ + where, + attributes: { exclude: modelExcludedFields }, + transaction + }) + } +} + +const instance = new FleetModelManager() +module.exports = instance diff --git a/src/helpers/error-messages.js b/src/helpers/error-messages.js index 9f4f7af1..2d629bfb 100644 --- a/src/helpers/error-messages.js +++ b/src/helpers/error-messages.js @@ -57,6 +57,10 @@ module.exports = { USER_ALREADY_ACTIVATED: 'User is already activated.', USER_NOT_ACTIVATED_YET: 'User is not activated yet.', REGISTRY_IS_SYSTEM: 'Registry is system and can\'t be updated or deleted', + REGISTRY_PRIVATE_OCI_CREDENTIALS_REQUIRED: 'Private OCI registries require username and password', + REGISTRY_PRIVATE_HF_TOKEN_REQUIRED: 'Private Hugging Face registries require a password token', + REGISTRY_INVALID_CA: 'Registry CA must be valid base64 when provided', + REGISTRY_NOT_OCI_FOR_IMAGE: 'Registry \'{}\' cannot be used for container images; type must be oci', INVALID_CLUSTER_CONTROLLER_UUID: 'Invalid cluster controller UUID \'{}\'', CLUSTER_CONTROLLER_NOT_FOUND: 'Cluster controller not found', CLUSTER_CONTROLLER_REGISTRATION_FAILED: 'Cluster controller registration failed', @@ -132,5 +136,49 @@ module.exports = { CONFIGMAP_IMMUTABLE: 'Configmap {} is immutable and cannot be updated. If you want to update it, please delete it and create a new configmap.', VOLUME_MOUNT_NOT_FOUND: 'Volume mount with name {} not found', INVALID_VOLUME_MOUNT_REFERENCE_FOR_VOLUME_MAPPING: 'Invalid volume mount reference for volume mapping: {}', - SECRET_TYPE_MISMATCH: 'Secret type mismatch. Secret {} is of type {} but trying to update with type {}' + SECRET_TYPE_MISMATCH: 'Secret type mismatch. Secret {} is of type {} but trying to update with type {}', + MODEL_NOT_FOUND: 'Model with name {} not found', + MODEL_ALREADY_EXISTS: 'Model with name {} already exists', + MODEL_IN_USE: 'Cannot delete model \'{}\': bound by microservices {}', + MODEL_UNLINK_IN_USE: 'Cannot unlink model \'{}\' from agent {}: bound by microservices {}', + MODEL_HF_FILES_REQUIRED: 'Hugging Face models require a non-empty files list', + MODEL_OCI_FILES_FORBIDDEN: 'OCI models must not specify files', + RUNTIME_CLASS_NOT_FOUND: 'RuntimeClass with name {} not found', + RUNTIME_CLASS_ALREADY_EXISTS: 'RuntimeClass with name {} already exists', + RUNTIME_CLASS_IN_USE: 'Cannot delete RuntimeClass \'{}\': pinned by microservices {}', + RUNTIME_CLASS_UNLINK_IN_USE: 'Cannot unlink RuntimeClass \'{}\' from agent {}: pinned by microservices {}', + RUNTIME_CLASS_LINK_REQUIRES_EDGELET: 'Cannot link RuntimeClass to agent {}: container engine must be edgelet', + RUNTIME_CLASS_MISSING: 'RuntimeClass \'{}\' does not exist', + MICROSERVICE_CATALOG_BIND_PATH_REQUIRED: 'Model catalog bindPath is required when items are present', + MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE: 'Model catalog bindPath must be an absolute container path', + MICROSERVICE_CATALOG_PERMISSIONS: 'Model catalog permissions must be ro or rw', + MICROSERVICE_CATALOG_DUPLICATE_ITEM: 'Model catalog item name \'{}\' is duplicated', + MICROSERVICE_CATALOG_ITEM_NAME: 'Model catalog item name \'{}\' is invalid', + MICROSERVICE_CATALOG_PATH_COLLISION: 'Model catalog path \'{}\' collides with a volume or tmpfs mount', + MICROSERVICE_CATALOG_MODEL_NOT_FOUND: 'Model catalog item \'{}\' does not match a fleet model', + MICROSERVICE_RUN_AS_USER_GROUP_CONFLICT: 'runAsUser cannot contain \':\' when runAsGroup is set', + MICROSERVICE_MEMORY_SWAP_REQUIRES_LIMIT: 'memorySwap requires memoryLimit unless it is -1', + MICROSERVICE_MEMORY_SWAP_VALUE: 'memorySwap must be -1 or a positive MiB total', + MICROSERVICE_WORKING_DIR_ABSOLUTE: 'workingDir must be an absolute container path', + MICROSERVICE_POSITIVE_NUMBER: 'Field \'{}\' must be greater than 0', + MICROSERVICE_ARGV_TYPE: 'Field \'{}\' must be an array of strings', + MICROSERVICE_SYSCTL_UNKNOWN: 'sysctl \'{}\' is not allowed', + MICROSERVICE_SYSCTL_SHAPE: 'sysctls must be an object of string values', + MICROSERVICE_SYSCTL_HOST_NETWORK: 'net.* sysctls cannot be set when hostNetworkMode is true', + MICROSERVICE_SYSCTL_HOST_IPC: 'IPC sysctls cannot be set when ipcMode is host', + MICROSERVICE_ULIMIT_UNKNOWN: 'ulimit \'{}\' is not allowed', + MICROSERVICE_ULIMIT_SHAPE: 'ulimit \'{}\' must be an object with soft and hard', + MICROSERVICE_ULIMIT_SHAPE_ROOT: 'ulimits must be an object of {soft, hard} entries', + MICROSERVICE_ULIMIT_RANGE: 'ulimit \'{}\' soft must be <= hard unless unlimited', + MICROSERVICE_DEVICES_SHAPE: 'devices must be an array of {hostPath, containerPath, permissions} objects', + MICROSERVICE_DEVICE_HOST_PATH: 'device hostPath must be an absolute path under /dev', + MICROSERVICE_DEVICE_CONTAINER_PATH: 'device containerPath is required', + MICROSERVICE_DEVICE_PERMISSIONS: 'device permissions must be a combination of r, w, and m', + MICROSERVICE_TMPFS_SHAPE: 'tmpfs must be an array of {containerPath, size?, mode?} objects', + MICROSERVICE_TMPFS_PATH: 'tmpfs containerPath must be an absolute container path', + MICROSERVICE_TMPFS_SIZE: 'tmpfs size must be greater than 0', + MICROSERVICE_TEMPLATE_NOT_FOUND: 'Microservice template with name {} not found', + MICROSERVICE_TEMPLATE_ALREADY_EXISTS: 'Microservice template with name {} already exists', + MICROSERVICE_TEMPLATE_INVALID: 'Microservice template {} is invalid', + MICROSERVICE_TEMPLATE_FILE_NOT_FOUND: 'Microservice template file missing' } diff --git a/src/routes/models.js b/src/routes/models.js new file mode 100644 index 00000000..acc4ab32 --- /dev/null +++ b/src/routes/models.js @@ -0,0 +1,187 @@ +const constants = require('../helpers/constants') +const ModelController = require('../controllers/model-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/models', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.listModelsEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/models/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.getModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/models/:name', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.updateModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/models/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.deleteModelEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/models', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.createModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/models/yaml', + fileInput: 'model', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.createModelYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/models/yaml/:name', + fileInput: 'model', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.upsertModelYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'get', + path: '/api/v3/models/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.getModelLinkEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/models/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.linkModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/models/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.unlinkModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/schemas/model.js b/src/schemas/model.js new file mode 100644 index 00000000..8c455ff2 --- /dev/null +++ b/src/schemas/model.js @@ -0,0 +1,101 @@ +const { serviceNameRegex } = require('./utils/utils') + +const modelFormats = ['gguf', 'safetensors', 'onnx', 'pytorch', 'tensorrt', 'unknown'] + +const modelCreate = { + id: '/modelCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: 'string', + enum: modelFormats + } + }, + required: ['name', 'repo', 'registryId'], + additionalProperties: false +} + +const modelUpdate = { + id: '/modelUpdate', + type: 'object', + properties: { + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: 'string', + enum: modelFormats + } + }, + additionalProperties: false +} + +const modelLink = { + id: '/modelLink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +const modelUnlink = { + id: '/modelUnlink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +module.exports = { + mainSchemas: [modelCreate, modelUpdate, modelLink, modelUnlink], + innerSchemas: [modelCreate, modelUpdate] +} diff --git a/src/services/model-service.js b/src/services/model-service.js new file mode 100644 index 00000000..10457a68 --- /dev/null +++ b/src/services/model-service.js @@ -0,0 +1,271 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const FleetModelManager = require('../data/managers/fleet-model-manager') +const RegistryManager = require('../data/managers/registry-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const MicroserviceModelItemManager = require('../data/managers/microservice-model-item-manager') +const FogManager = require('../data/managers/iofog-manager') +const ChangeTrackingService = require('./change-tracking-service') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function toWireModel (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + return { + uuid: plain.uuid, + name: plain.name, + repo: plain.repo, + revision: plain.revision != null ? plain.revision : '', + registryId: plain.registryId, + files: Array.isArray(plain.files) ? plain.files : [], + format: plain.format + } +} + +async function findLinkedFogUuids (model, transaction) { + if (!model || typeof model.getFogs !== 'function') { + return [] + } + const fogs = await model.getFogs({ transaction }) + return fogs.map((fog) => fog.uuid) +} + +async function _updateChangeTrackingForFogs (fogUuids, transaction) { + for (const fogUuid of fogUuids) { + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.models, transaction) + } +} + +async function findMicroservicesBindingModelName (modelName, options, transaction) { + const items = await MicroserviceModelItemManager.findAll({ name: modelName }, transaction) + const blocking = [] + for (const item of items) { + if (options && options.fogUuid) { + const microservice = await MicroserviceManager.findOne({ uuid: item.microserviceUuid }, transaction) + if (!microservice || microservice.iofogUuid !== options.fogUuid) { + continue + } + blocking.push(microservice.uuid) + continue + } + blocking.push(item.microserviceUuid) + } + return blocking +} + +async function _validateRegistryAndFiles (registryId, files, transaction) { + const registry = await RegistryManager.findOne({ id: registryId }, transaction) + if (!registry) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryId)) + } + const type = registry.type || 'oci' + const fileList = Array.isArray(files) ? files : [] + if (type === 'hf') { + if (fileList.length === 0) { + throw new Errors.ValidationError(ErrorMessages.MODEL_HF_FILES_REQUIRED) + } + } else if (fileList.length > 0) { + throw new Errors.ValidationError(ErrorMessages.MODEL_OCI_FILES_FORBIDDEN) + } + return registry +} + +async function getModelByName (name, transaction) { + const model = await FleetModelManager.findOne({ name }, transaction) + if (!model) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.MODEL_NOT_FOUND, name)) + } + return model +} + +async function listModelsEndpoint (transaction) { + const models = await FleetModelManager.findAll({}, transaction) + return { + models: models.map(toWireModel) + } +} + +async function getModelEndpoint (name, transaction) { + const model = await getModelByName(name, transaction) + return toWireModel(model) +} + +async function upsertModelEndpoint (name, data, transaction) { + const existing = await FleetModelManager.findOne({ name }, transaction) + if (existing) { + return updateModelEndpoint(name, data, transaction) + } + return createModelEndpoint({ ...data, name }, transaction) +} + +async function createModelEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.modelCreate) + + const existing = await FleetModelManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MODEL_ALREADY_EXISTS, data.name)) + } + + const files = Array.isArray(data.files) ? data.files : [] + await _validateRegistryAndFiles(data.registryId, files, transaction) + + const created = await FleetModelManager.create({ + uuid: AppHelper.generateUUID(), + name: data.name, + repo: data.repo, + revision: data.revision != null ? data.revision : '', + registryId: data.registryId, + files, + format: data.format + }, transaction) + + return toWireModel(created) +} + +async function updateModelEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.modelUpdate) + const model = await getModelByName(name, transaction) + + const nextRepo = data.repo !== undefined ? data.repo : model.repo + const nextRevision = data.revision !== undefined ? data.revision : (model.revision != null ? model.revision : '') + const nextRegistryId = data.registryId !== undefined ? data.registryId : model.registryId + const nextFiles = data.files !== undefined ? data.files : (Array.isArray(model.files) ? model.files : []) + const nextFormat = data.format !== undefined ? data.format : model.format + + await _validateRegistryAndFiles(nextRegistryId, nextFiles, transaction) + + const linkedFogUuids = await findLinkedFogUuids(model, transaction) + + await FleetModelManager.update({ name }, { + repo: nextRepo, + revision: nextRevision, + registryId: nextRegistryId, + files: nextFiles, + format: nextFormat + }, transaction) + + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + + return getModelEndpoint(name, transaction) +} + +async function deleteModelEndpoint (name, transaction) { + const model = await getModelByName(name, transaction) + const blocking = await findMicroservicesBindingModelName(name, {}, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MODEL_IN_USE, name, blocking.join(', '))) + } + + const linkedFogUuids = await findLinkedFogUuids(model, transaction) + await FleetModelManager.delete({ name }, transaction) + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + return {} +} + +async function getModelLinkEndpoint (name, transaction) { + const model = await getModelByName(name, transaction) + return { + fogUuids: await findLinkedFogUuids(model, transaction) + } +} + +async function linkModelEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.modelLink) + const model = await getModelByName(name, transaction) + const alreadyLinked = new Set(await findLinkedFogUuids(model, transaction)) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + await agent.addModel(model, { transaction }) + } + + const newlyLinked = fogUuids.filter((uuid) => !alreadyLinked.has(uuid)) + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs(newlyLinked, transaction) + } + + return getModelEndpoint(name, transaction) +} + +async function unlinkModelEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.modelUnlink) + const model = await getModelByName(name, transaction) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + const blocking = await findMicroservicesBindingModelName(name, { fogUuid }, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MODEL_UNLINK_IN_USE, name, fogUuid, blocking.join(', '))) + } + await agent.removeModel(model, { transaction }) + } + + await _updateChangeTrackingForFogs(fogUuids, transaction) + return {} +} + +async function ensureModelsLinkedToFog (fogUuid, modelNames, transaction) { + const names = [...new Set((modelNames || []).filter((name) => typeof name === 'string' && name.length > 0))] + if (names.length === 0) { + return [] + } + + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + + const newlyLinked = [] + for (const name of names) { + const model = await getModelByName(name, transaction) + const alreadyLinked = typeof agent.hasModel === 'function' + ? await agent.hasModel(model, { transaction }) + : false + if (alreadyLinked) { + continue + } + await agent.addModel(model, { transaction }) + newlyLinked.push(name) + } + + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs([fogUuid], transaction) + } + + return newlyLinked +} + +module.exports = { + listModelsEndpoint: TransactionDecorator.generateTransaction(listModelsEndpoint), + getModelEndpoint: TransactionDecorator.generateTransaction(getModelEndpoint), + createModelEndpoint: TransactionDecorator.generateTransaction(createModelEndpoint), + updateModelEndpoint: TransactionDecorator.generateTransaction(updateModelEndpoint), + deleteModelEndpoint: TransactionDecorator.generateTransaction(deleteModelEndpoint), + getModelLinkEndpoint: TransactionDecorator.generateTransaction(getModelLinkEndpoint), + linkModelEndpoint: TransactionDecorator.generateTransaction(linkModelEndpoint), + unlinkModelEndpoint: TransactionDecorator.generateTransaction(unlinkModelEndpoint), + upsertModelEndpoint: TransactionDecorator.generateTransaction(upsertModelEndpoint), + ensureModelsLinkedToFog: TransactionDecorator.generateTransaction(ensureModelsLinkedToFog), + toWireModel +} diff --git a/src/services/yaml-parser-service.js b/src/services/yaml-parser-service.js index cda0c3c9..c0296fc8 100644 --- a/src/services/yaml-parser-service.js +++ b/src/services/yaml-parser-service.js @@ -1,4 +1,5 @@ const Errors = require('../helpers/errors') +const CatalogContainer = require('../helpers/microservice-container-catalog') const lget = require('lodash/get') const yaml = require('js-yaml') @@ -253,6 +254,23 @@ const parseMicroserviceImages = async (fileImages) => { return { registryId, catalogItemId: undefined, images } } +function parseMicroserviceTemplateRef (template) { + if (template == null) { + return undefined + } + if (typeof template !== 'object' || Array.isArray(template)) { + throw new Errors.ValidationError('Invalid template format') + } + if (!template.name) { + throw new Errors.ValidationError('template.name is required when template is specified') + } + const result = { name: template.name } + if (template.variables !== undefined) { + result.variables = template.variables + } + return result +} + const parseMicroserviceYAML = async (microservice) => { const { registryId, catalogItemId, images } = await parseMicroserviceImages(microservice.images) const container = microservice.container || {} @@ -324,13 +342,17 @@ const parseMicroserviceYAML = async (microservice) => { }) } + const commands = container.commands !== undefined ? container.commands : container.cmd + const sysctls = container.sysctls !== undefined + ? CatalogContainer.normalizeSysctls(container.sysctls) + : undefined + const microserviceData = { config: microservice.config != null ? JSON.stringify(microservice.config) : undefined, name: microservice.name, catalogItemId, agentName: lget(microservice, 'agent.name'), registryId, - ...container, hostNetworkMode: lget(microservice, 'container.hostNetworkMode', false), isPrivileged: lget(microservice, 'container.isPrivileged', false), pidMode: lget(microservice, 'container.pidMode', ''), @@ -343,15 +365,34 @@ const parseMicroserviceYAML = async (microservice) => { capDrop: lget(microservice, 'container.capDrop', []), ports: (lget(microservice, 'container.ports', [])), volumeMappings: lget(microservice, 'container.volumes', []), - cmd: lget(microservice, 'container.commands', []), + commands, + cmd: commands, env: parseEnvVariables(lget(microservice, 'container.env', [])), images, extraHosts: lget(microservice, 'container.extraHosts', []), application: microservice.application, + models: microservice.models, + template: parseMicroserviceTemplateRef(microservice.template), schedule: lget(microservice, 'schedule', 50), serviceAccount: lget(microservice, 'serviceAccount', undefined), natsEnabled: lget(microservice, 'natsEnabled', undefined), - natsConfig: lget(microservice, 'natsConfig', undefined) + natsConfig: lget(microservice, 'natsConfig', undefined), + runAsUser: lget(microservice, 'container.runAsUser', undefined), + runAsGroup: lget(microservice, 'container.runAsGroup', undefined), + readOnlyRootFilesystem: lget(microservice, 'container.readOnlyRootFilesystem', undefined), + platform: lget(microservice, 'container.platform', undefined), + runtime: lget(microservice, 'container.runtime', undefined), + cdiDevices: lget(microservice, 'container.cdiDevices', undefined), + sysctls, + ulimits: lget(microservice, 'container.ulimits', undefined), + cpus: lget(microservice, 'container.cpus', undefined), + memoryReservation: lget(microservice, 'container.memoryReservation', undefined), + memorySwap: lget(microservice, 'container.memorySwap', undefined), + shmSize: lget(microservice, 'container.shmSize', undefined), + devices: lget(microservice, 'container.devices', undefined), + tmpfs: lget(microservice, 'container.tmpfs', undefined), + workingDir: lget(microservice, 'container.workingDir', undefined), + entrypoint: lget(microservice, 'container.entrypoint', undefined) } _deleteUndefinedFields(microserviceData) return microserviceData @@ -648,6 +689,162 @@ async function parseNatsUserRuleFile (fileContent, options = {}) { } } +async function parseModelFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'Model') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null || doc.spec == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata or spec') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.modelName) { + if (doc.metadata.name !== options.modelName) { + throw new Errors.ValidationError(`Model name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.modelName})`) + } + } + + const spec = doc.spec || {} + const registryId = spec.registryId != null ? spec.registryId : spec.registry + const result = { + name: lget(doc, 'metadata.name', undefined), + repo: spec.repo, + revision: spec.revision, + registryId, + files: spec.files, + format: spec.format + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.modelName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + +async function parseRuntimeClassFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'RuntimeClass') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.runtimeClassName) { + if (doc.metadata.name !== options.runtimeClassName) { + throw new Errors.ValidationError(`RuntimeClass name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.runtimeClassName})`) + } + } + + const handler = doc.handler != null && doc.handler !== '' + ? doc.handler + : lget(doc, 'spec.handler', undefined) + if (handler == null || handler === '') { + throw new Errors.ValidationError('Invalid YAML format: handler is required') + } + + const result = { + name: lget(doc, 'metadata.name', undefined), + handler + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.runtimeClassName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + +async function parseMicroserviceTemplateFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'MicroserviceTemplate') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null || doc.spec == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata or spec') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.templateName) { + if (doc.metadata.name !== options.templateName) { + throw new Errors.ValidationError(`MicroserviceTemplate name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.templateName})`) + } + } + + const spec = doc.spec || {} + if (!spec.microservice || typeof spec.microservice !== 'object' || Array.isArray(spec.microservice)) { + throw new Errors.ValidationError('MicroserviceTemplate spec.microservice is required') + } + + const microservice = await parseMicroserviceYAML(spec.microservice) + delete microservice.name + delete microservice.application + delete microservice.iofogUuid + delete microservice.agentName + delete microservice.flowId + + const result = { + name: lget(doc, 'metadata.name', undefined), + description: spec.description, + variables: spec.variables, + microservice + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.templateName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + module.exports = { parseAppTemplateFile, parseAppFile, @@ -661,5 +858,8 @@ module.exports = { parseServiceFile, parseRoleFile, parseRoleBindingFile, - parseServiceAccountFile + parseServiceAccountFile, + parseModelFile, + parseRuntimeClassFile, + parseMicroserviceTemplateFile } diff --git a/test/src/routes/models-routes.test.js b/test/src/routes/models-routes.test.js new file mode 100644 index 00000000..e3db8e48 --- /dev/null +++ b/test/src/routes/models-routes.test.js @@ -0,0 +1,19 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Model routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/models.js'), 'utf8') + + it('registers user model CRUD, YAML, and link routes', () => { + expect(source).to.include("path: '/api/v3/models'") + expect(source).to.include("path: '/api/v3/models/:name'") + expect(source).to.include("path: '/api/v3/models/yaml'") + expect(source).to.include("path: '/api/v3/models/yaml/:name'") + expect(source).to.include("path: '/api/v3/models/:name/link'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + }) +}) diff --git a/test/src/services/model-service.test.js b/test/src/services/model-service.test.js new file mode 100644 index 00000000..bebc0494 --- /dev/null +++ b/test/src/services/model-service.test.js @@ -0,0 +1,327 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const FleetModelManager = require('../../../src/data/managers/fleet-model-manager') +const RegistryManager = require('../../../src/data/managers/registry-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceModelItemManager = require('../../../src/data/managers/microservice-model-item-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const ModelService = require('../../../src/services/model-service') +const AppHelper = require('../../../src/helpers/app-helper') +const Errors = require('../../../src/helpers/errors') + +describe('Model Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + const modelUuid = '3f2c8a1e-2b64-4c0d-9f11-0a1b2c3d4e5f' + const fogUuid = 'fog-1' + + afterEach(() => { + $sandbox.restore() + }) + + function stubModelRow (overrides = {}) { + const row = { + uuid: modelUuid, + name: 'test-model', + repo: 'org/repo', + revision: '', + registryId: 3, + files: ['file.gguf'], + format: 'gguf', + getFogs: $sandbox.stub().resolves([]), + ...overrides + } + return row + } + + describe('.createModelEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(AppHelper, 'generateUUID').returns(modelUuid) + $sandbox.stub(FleetModelManager, 'findOne').resolves(null) + $sandbox.stub(FleetModelManager, 'create').callsFake(async (data) => data) + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + }) + + it('creates a model with a server-generated uuid', async () => { + const result = await ModelService.createModelEndpoint({ + name: 'test-model', + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }, transaction) + + expect(result).to.eql({ + uuid: modelUuid, + name: 'test-model', + repo: 'org/repo', + revision: '', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }) + expect(FleetModelManager.create).to.have.been.calledOnce + }) + + it('rejects a duplicate name', async () => { + FleetModelManager.findOne.resolves(stubModelRow()) + + await expect(ModelService.createModelEndpoint({ + name: 'test-model', + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'] + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + + it('requires files for a Hugging Face registry', async () => { + await expect(ModelService.createModelEndpoint({ + name: 'test-model', + repo: 'org/repo', + registryId: 3 + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /non-empty files list/) + }) + + it('rejects files for an OCI registry', async () => { + RegistryManager.findOne.resolves({ id: 1, type: 'oci' }) + + await expect(ModelService.createModelEndpoint({ + name: 'oci-model', + repo: 'org/repo', + registryId: 1, + files: ['weights.bin'] + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /must not specify files/) + }) + }) + + describe('.listModelsEndpoint() and .getModelEndpoint()', () => { + const row = { + uuid: modelUuid, + name: 'test-model', + repo: 'org/repo', + revision: 'abc', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + } + + beforeEach(() => { + $sandbox.stub(FleetModelManager, 'findAll').resolves([row]) + $sandbox.stub(FleetModelManager, 'findOne').resolves(row) + }) + + it('lists all fleet models', async () => { + const result = await ModelService.listModelsEndpoint(transaction) + expect(result).to.eql({ models: [row] }) + }) + + it('returns spec only without linked fogs', async () => { + const result = await ModelService.getModelEndpoint('test-model', transaction) + expect(result).to.eql(row) + expect(result).to.not.have.property('fogUuids') + expect(result).to.not.have.property('fogs') + }) + + it('throws when the model is missing', async () => { + FleetModelManager.findOne.resolves(null) + await expect(ModelService.getModelEndpoint('missing', transaction)) + .to.be.rejectedWith(Errors.NotFoundError, /missing/) + }) + }) + + describe('.updateModelEndpoint()', () => { + beforeEach(() => { + const row = stubModelRow() + $sandbox.stub(FleetModelManager, 'findOne').resolves(row) + $sandbox.stub(FleetModelManager, 'update').resolves() + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('updates spec fields and notifies linked agents', async () => { + FleetModelManager.findOne.resolves(stubModelRow({ + getFogs: $sandbox.stub().resolves([{ uuid: fogUuid }]) + })) + + await ModelService.updateModelEndpoint('test-model', { + revision: 'main', + files: ['other.gguf'] + }, transaction) + + expect(FleetModelManager.update).to.have.been.calledWith( + { name: 'test-model' }, + sinon.match({ + repo: 'org/repo', + revision: 'main', + registryId: 3, + files: ['other.gguf'], + format: 'gguf' + }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.models, + transaction + ) + }) + }) + + describe('.deleteModelEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetModelManager, 'findOne').resolves(stubModelRow()) + $sandbox.stub(FleetModelManager, 'delete').resolves() + $sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes when no microservice binds the name', async () => { + const result = await ModelService.deleteModelEndpoint('test-model', transaction) + expect(result).to.eql({}) + expect(FleetModelManager.delete).to.have.been.calledWith({ name: 'test-model' }, transaction) + }) + + it('refuses delete when any microservice binds the name', async () => { + MicroserviceModelItemManager.findAll.resolves([ + { microserviceUuid: 'ms-1', name: 'test-model' } + ]) + + await expect(ModelService.deleteModelEndpoint('test-model', transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-1/) + expect(FleetModelManager.delete).to.not.have.been.called + }) + }) + + describe('link and unlink', () => { + let model + let agent + + beforeEach(() => { + model = stubModelRow() + agent = { + uuid: fogUuid, + addModel: $sandbox.stub().resolves(), + removeModel: $sandbox.stub().resolves() + } + $sandbox.stub(FleetModelManager, 'findOne').resolves(model) + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + $sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links a fog and sets the models change flag', async () => { + const result = await ModelService.linkModelEndpoint('test-model', [fogUuid], transaction) + expect(agent.addModel).to.have.been.calledWith(model, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.models, + transaction + ) + expect(result.name).to.equal('test-model') + }) + + it('does not set the change flag when the fog is already linked', async () => { + model.getFogs.resolves([{ uuid: fogUuid }]) + await ModelService.linkModelEndpoint('test-model', [fogUuid], transaction) + expect(ChangeTrackingService.update).to.not.have.been.called + }) + + it('returns linked fog uuids', async () => { + model.getFogs.resolves([{ uuid: fogUuid }]) + const result = await ModelService.getModelLinkEndpoint('test-model', transaction) + expect(result).to.eql({ fogUuids: [fogUuid] }) + }) + + it('unlinks a fog and sets the models change flag', async () => { + await ModelService.unlinkModelEndpoint('test-model', [fogUuid], transaction) + expect(agent.removeModel).to.have.been.calledWith(model, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.models, + transaction + ) + }) + + it('refuses unlink when a microservice on that fog binds the name', async () => { + MicroserviceModelItemManager.findAll.resolves([ + { microserviceUuid: 'ms-bound', name: 'test-model' } + ]) + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ + uuid: 'ms-bound', + iofogUuid: fogUuid + }) + + await expect(ModelService.unlinkModelEndpoint('test-model', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-bound/) + expect(agent.removeModel).to.not.have.been.called + }) + }) + + describe('.upsertModelEndpoint()', () => { + it('creates when the name is new and updates when it already exists', async () => { + $sandbox.stub(AppHelper, 'generateUUID').returns(modelUuid) + $sandbox.stub(FleetModelManager, 'findOne').resolves(null) + $sandbox.stub(FleetModelManager, 'create').callsFake(async (data) => data) + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + + const created = await ModelService.upsertModelEndpoint('test-model', { + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'] + }, transaction) + expect(created.uuid).to.equal(modelUuid) + expect(FleetModelManager.create).to.have.been.calledOnce + + const existing = stubModelRow({ + getFogs: $sandbox.stub().resolves([]) + }) + FleetModelManager.findOne.resolves(existing) + $sandbox.stub(FleetModelManager, 'update').resolves() + + await ModelService.upsertModelEndpoint('test-model', { + repo: 'org/other', + files: ['file.gguf'] + }, transaction) + expect(FleetModelManager.update).to.have.been.calledOnce + expect(FleetModelManager.create).to.have.been.calledOnce + }) + }) + + describe('.ensureModelsLinkedToFog()', () => { + let model + let agent + + beforeEach(() => { + model = stubModelRow() + agent = { + uuid: fogUuid, + hasModel: $sandbox.stub().resolves(false), + addModel: $sandbox.stub().resolves() + } + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(FleetModelManager, 'findOne').resolves(model) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links missing models and is idempotent when already linked', async () => { + const first = await ModelService.ensureModelsLinkedToFog(fogUuid, ['test-model'], transaction) + expect(first).to.eql(['test-model']) + expect(agent.addModel).to.have.been.calledOnce + expect(ChangeTrackingService.update).to.have.been.calledOnce + + agent.hasModel.resolves(true) + ChangeTrackingService.update.resetHistory() + agent.addModel.resetHistory() + + const second = await ModelService.ensureModelsLinkedToFog(fogUuid, ['test-model'], transaction) + expect(second).to.eql([]) + expect(agent.addModel).to.not.have.been.called + expect(ChangeTrackingService.update).to.not.have.been.called + }) + }) +}) diff --git a/test/src/services/yaml-parser-service.test.js b/test/src/services/yaml-parser-service.test.js index 7c6fc0f6..086435b8 100644 --- a/test/src/services/yaml-parser-service.test.js +++ b/test/src/services/yaml-parser-service.test.js @@ -21,6 +21,33 @@ spec: natsRule: 'app-rule' }) }) + + it('parses nested microservice serviceAccount', async () => { + const yaml = ` +kind: Application +metadata: + name: app-a +spec: + microservices: + - name: ms-a + serviceAccount: + roleRef: + kind: Role + name: custom-role + apiGroup: edgelet.iofog.org/v1 + container: + env: [] +` + const result = await YamlParserService.parseAppFile(yaml) + expect(result.microservices).to.have.length(1) + expect(result.microservices[0].serviceAccount).to.eql({ + roleRef: { + kind: 'Role', + name: 'custom-role', + apiGroup: 'edgelet.iofog.org/v1' + } + }) + }) }) describe('.parseMicroserviceFile()', () => { @@ -45,6 +72,30 @@ spec: expect(result.name).to.eql('ms-a') }) + it('parses microservice serviceAccount', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-a +spec: + serviceAccount: + roleRef: + kind: Role + name: custom-role + apiGroup: edgelet.iofog.org/v1 + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.serviceAccount).to.eql({ + roleRef: { + kind: 'Role', + name: 'custom-role', + apiGroup: 'edgelet.iofog.org/v1' + } + }) + }) + it('parses microservice natsEnabled', async () => { const yaml = ` kind: Microservice @@ -84,6 +135,81 @@ spec: ]) }) + it('maps spec.models and prefers container.commands over cmd', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-models +spec: + models: + bindPath: /models + permissions: ro + items: + - name: test-model + container: + commands: + - python + - app.py + cmd: + - ignored + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(result.commands).to.eql(['python', 'app.py']) + expect(result.cmd).to.eql(['python', 'app.py']) + expect(result).to.not.have.property('volumes') + }) + + it('maps container devices, tmpfs, sysctls, and ulimits', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-container +spec: + container: + sysctls: + net.ipv4.tcp_syncookies: 1 + ulimits: + nofile: + soft: 65536 + hard: 65536 + devices: + - hostPath: /dev/null + containerPath: /dev/null + permissions: rwm + tmpfs: + - containerPath: /tmp + size: 64 + volumes: + - hostDestination: /data + containerDestination: /data + accessMode: rw + type: bind + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.sysctls).to.eql({ 'net.ipv4.tcp_syncookies': '1' }) + expect(result.ulimits).to.eql({ nofile: { soft: 65536, hard: 65536 } }) + expect(result.devices).to.eql([{ + hostPath: '/dev/null', + containerPath: '/dev/null', + permissions: 'rwm' + }]) + expect(result.tmpfs).to.eql([{ containerPath: '/tmp', size: 64 }]) + expect(result.volumeMappings).to.eql([{ + hostDestination: '/data', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind' + }]) + expect(result).to.not.have.property('volumes') + }) + it('does not map deprecated top-level natsAccess', async () => { const yaml = ` kind: Microservice @@ -97,6 +223,72 @@ spec: const result = await YamlParserService.parseMicroserviceFile(yaml) expect(result.natsAccess).to.eql(undefined) }) + + it('maps spec.template with object variables', async () => { + const yaml = ` +kind: Microservice +metadata: + name: whisper-instance-1 +spec: + application: test-app + agent: + name: edge-node-1 + template: + name: whisper-infer + variables: + model: llama-7b + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.name).to.eql('whisper-instance-1') + expect(result.application).to.eql('test-app') + expect(result.agentName).to.eql('edge-node-1') + expect(result.template).to.eql({ + name: 'whisper-infer', + variables: { model: 'llama-7b' } + }) + }) + + it('maps spec.template with array variables', async () => { + const yaml = ` +kind: Microservice +metadata: + name: test-app/whisper-instance-2 +spec: + template: + name: whisper-infer + variables: + - key: model + value: llama-7b + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.name).to.eql('whisper-instance-2') + expect(result.application).to.eql('test-app') + expect(result.template).to.eql({ + name: 'whisper-infer', + variables: [{ key: 'model', value: 'llama-7b' }] + }) + }) + + it('rejects spec.template without name', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-template +spec: + template: + variables: + model: llama-7b + container: + env: [] +` + await expect(YamlParserService.parseMicroserviceFile(yaml)).to.be.rejectedWith( + 'template.name is required when template is specified' + ) + }) }) describe('NATS rule YAML parsing', () => { @@ -136,4 +328,233 @@ spec: }) }) }) + + describe('.parseModelFile()', () => { + it('parses metadata.name and spec fields matching the wire object', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: Model +metadata: + name: test-model +spec: + repo: org/repo + revision: abc123 + registryId: 3 + files: + - file.gguf + format: gguf +` + const result = await YamlParserService.parseModelFile(yaml) + expect(result).to.eql({ + name: 'test-model', + repo: 'org/repo', + revision: 'abc123', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }) + }) + + it('accepts spec.registry as an alias for registryId', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: Model +metadata: + name: test-model +spec: + repo: org/repo + registry: 3 + files: + - file.gguf +` + const result = await YamlParserService.parseModelFile(yaml) + expect(result.registryId).to.equal(3) + }) + + it('omits name on update and rejects a path mismatch', async () => { + const yaml = ` +kind: Model +metadata: + name: test-model +spec: + repo: org/repo + registryId: 3 +` + const updated = await YamlParserService.parseModelFile(yaml, { + isUpdate: true, + modelName: 'test-model' + }) + expect(updated).to.not.have.property('name') + expect(updated.repo).to.equal('org/repo') + + await expect(YamlParserService.parseModelFile(yaml, { + isUpdate: true, + modelName: 'other-model' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) + + describe('.parseRuntimeClassFile()', () => { + it('parses metadata.name and top-level handler matching the wire object', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: RuntimeClass +metadata: + name: spin +handler: spin +` + const result = await YamlParserService.parseRuntimeClassFile(yaml) + expect(result).to.eql({ + name: 'spin', + handler: 'spin' + }) + }) + + it('accepts iofog.org/v3 apiVersion', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: RuntimeClass +metadata: + name: nvidia-cdi +handler: nvidia-cdi +` + const result = await YamlParserService.parseRuntimeClassFile(yaml) + expect(result.handler).to.equal('nvidia-cdi') + }) + + it('accepts spec.handler as a fallback', async () => { + const yaml = ` +kind: RuntimeClass +metadata: + name: spin +spec: + handler: spin +` + const result = await YamlParserService.parseRuntimeClassFile(yaml) + expect(result).to.eql({ name: 'spin', handler: 'spin' }) + }) + + it('rejects YAML without handler', async () => { + const yaml = ` +kind: RuntimeClass +metadata: + name: spin +` + await expect(YamlParserService.parseRuntimeClassFile(yaml)) + .to.be.rejectedWith(/handler is required/) + }) + + it('omits name on update and rejects a path mismatch', async () => { + const yaml = ` +kind: RuntimeClass +metadata: + name: spin +handler: wasmtime +` + const updated = await YamlParserService.parseRuntimeClassFile(yaml, { + isUpdate: true, + runtimeClassName: 'spin' + }) + expect(updated).to.not.have.property('name') + expect(updated.handler).to.equal('wasmtime') + + await expect(YamlParserService.parseRuntimeClassFile(yaml, { + isUpdate: true, + runtimeClassName: 'other-class' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) + + describe('.parseMicroserviceTemplateFile()', () => { + it('parses kind, variables, and microservice catalog/container fields', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: MicroserviceTemplate +metadata: + name: nginx-edge +spec: + description: nginx at the edge + variables: + - key: port + description: listen port + defaultValue: "8080" + microservice: + images: + x86: nginx:alpine + models: + bindPath: /models + permissions: ro + items: + - name: test-model + container: + commands: + - nginx + - -g + - daemon off; + env: + - key: PORT + value: "{{ port }}" +` + const result = await YamlParserService.parseMicroserviceTemplateFile(yaml) + expect(result.name).to.equal('nginx-edge') + expect(result.description).to.equal('nginx at the edge') + expect(result.variables).to.eql([ + { key: 'port', description: 'listen port', defaultValue: '8080' } + ]) + expect(result.microservice.images).to.eql([ + { archId: 1, containerImage: 'nginx:alpine' } + ]) + expect(result.microservice.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) + expect(result.microservice.env).to.eql([{ key: 'PORT', value: '{{ port }}' }]) + expect(result.microservice).to.not.have.property('name') + expect(result.microservice).to.not.have.property('application') + }) + + it('accepts iofog.org/v3 apiVersion', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: MicroserviceTemplate +metadata: + name: nginx-edge +spec: + microservice: + images: + x86: nginx:alpine + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceTemplateFile(yaml) + expect(result.name).to.equal('nginx-edge') + }) + + it('omits name on update and rejects a path mismatch', async () => { + const yaml = ` +kind: MicroserviceTemplate +metadata: + name: nginx-edge +spec: + microservice: + images: + x86: nginx:alpine + container: + env: [] +` + const updated = await YamlParserService.parseMicroserviceTemplateFile(yaml, { + isUpdate: true, + templateName: 'nginx-edge' + }) + expect(updated).to.not.have.property('name') + expect(updated.microservice.images[0].containerImage).to.equal('nginx:alpine') + + await expect(YamlParserService.parseMicroserviceTemplateFile(yaml, { + isUpdate: true, + templateName: 'other-template' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) }) From 1f194df1b8e6eac474671564f1265d84ea5695e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:50:41 +0300 Subject: [PATCH 05/55] Add RuntimeClass catalog with edgelet-only fog attach. Deploy validates runtime against available runtimes and auto-links the class on edgelet agents. --- src/controllers/runtime-class-controller.js | 63 ++++ src/data/managers/runtime-class-manager.js | 12 + src/routes/runtimeClasses.js | 187 ++++++++++++ src/schemas/runtime-class.js | 73 +++++ src/services/runtime-class-service.js | 247 +++++++++++++++ test/src/routes/runtimeClasses-routes.test.js | 19 ++ .../services/runtime-class-service.test.js | 285 ++++++++++++++++++ 7 files changed, 886 insertions(+) create mode 100644 src/controllers/runtime-class-controller.js create mode 100644 src/data/managers/runtime-class-manager.js create mode 100644 src/routes/runtimeClasses.js create mode 100644 src/schemas/runtime-class.js create mode 100644 src/services/runtime-class-service.js create mode 100644 test/src/routes/runtimeClasses-routes.test.js create mode 100644 test/src/services/runtime-class-service.test.js diff --git a/src/controllers/runtime-class-controller.js b/src/controllers/runtime-class-controller.js new file mode 100644 index 00000000..6ae14587 --- /dev/null +++ b/src/controllers/runtime-class-controller.js @@ -0,0 +1,63 @@ +const RuntimeClassService = require('../services/runtime-class-service') +const YAMLParserService = require('../services/yaml-parser-service') + +const listRuntimeClassesEndpoint = async (req) => { + return RuntimeClassService.listRuntimeClassesEndpoint() +} + +const getRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.getRuntimeClassEndpoint(req.params.name) +} + +const createRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.createRuntimeClassEndpoint(req.body) +} + +const updateRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.updateRuntimeClassEndpoint(req.params.name, req.body) +} + +const deleteRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.deleteRuntimeClassEndpoint(req.params.name) +} + +const createRuntimeClassYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const runtimeClassData = await YAMLParserService.parseRuntimeClassFile(fileContent) + return RuntimeClassService.createRuntimeClassEndpoint(runtimeClassData) +} + +const upsertRuntimeClassYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const name = req.params.name + const runtimeClassData = await YAMLParserService.parseRuntimeClassFile(fileContent, { + isUpdate: true, + runtimeClassName: name + }) + return RuntimeClassService.upsertRuntimeClassEndpoint(name, runtimeClassData) +} + +const getRuntimeClassLinkEndpoint = async (req) => { + return RuntimeClassService.getRuntimeClassLinkEndpoint(req.params.name) +} + +const linkRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.linkRuntimeClassEndpoint(req.params.name, req.body.fogUuids) +} + +const unlinkRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.unlinkRuntimeClassEndpoint(req.params.name, req.body.fogUuids) +} + +module.exports = { + listRuntimeClassesEndpoint, + getRuntimeClassEndpoint, + createRuntimeClassEndpoint, + updateRuntimeClassEndpoint, + deleteRuntimeClassEndpoint, + createRuntimeClassYamlEndpoint, + upsertRuntimeClassYamlEndpoint, + getRuntimeClassLinkEndpoint, + linkRuntimeClassEndpoint, + unlinkRuntimeClassEndpoint +} diff --git a/src/data/managers/runtime-class-manager.js b/src/data/managers/runtime-class-manager.js new file mode 100644 index 00000000..f12e266f --- /dev/null +++ b/src/data/managers/runtime-class-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const RuntimeClass = models.RuntimeClass + +class RuntimeClassManager extends BaseManager { + getEntity () { + return RuntimeClass + } +} + +const instance = new RuntimeClassManager() +module.exports = instance diff --git a/src/routes/runtimeClasses.js b/src/routes/runtimeClasses.js new file mode 100644 index 00000000..cc070ed7 --- /dev/null +++ b/src/routes/runtimeClasses.js @@ -0,0 +1,187 @@ +const constants = require('../helpers/constants') +const RuntimeClassController = require('../controllers/runtime-class-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/runtimeClasses', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.listRuntimeClassesEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/runtimeClasses/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.getRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/runtimeClasses/:name', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.updateRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/runtimeClasses/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.deleteRuntimeClassEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/runtimeClasses', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.createRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/runtimeClasses/yaml', + fileInput: 'runtimeClass', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.createRuntimeClassYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/runtimeClasses/yaml/:name', + fileInput: 'runtimeClass', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.upsertRuntimeClassYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'get', + path: '/api/v3/runtimeClasses/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.getRuntimeClassLinkEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/runtimeClasses/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.linkRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/runtimeClasses/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.unlinkRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/schemas/runtime-class.js b/src/schemas/runtime-class.js new file mode 100644 index 00000000..3d1fe25e --- /dev/null +++ b/src/schemas/runtime-class.js @@ -0,0 +1,73 @@ +const { serviceNameRegex, nameRegex } = require('./utils/utils') + +const runtimeClassCreate = { + id: '/runtimeClassCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + handler: { + type: 'string', + pattern: nameRegex, + minLength: 1, + maxLength: 253 + } + }, + required: ['name', 'handler'], + additionalProperties: false +} + +const runtimeClassUpdate = { + id: '/runtimeClassUpdate', + type: 'object', + properties: { + handler: { + type: 'string', + pattern: nameRegex, + minLength: 1, + maxLength: 253 + } + }, + additionalProperties: false +} + +const runtimeClassLink = { + id: '/runtimeClassLink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +const runtimeClassUnlink = { + id: '/runtimeClassUnlink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +module.exports = { + mainSchemas: [runtimeClassCreate, runtimeClassUpdate, runtimeClassLink, runtimeClassUnlink], + innerSchemas: [runtimeClassCreate, runtimeClassUpdate] +} diff --git a/src/services/runtime-class-service.js b/src/services/runtime-class-service.js new file mode 100644 index 00000000..2f5c6be3 --- /dev/null +++ b/src/services/runtime-class-service.js @@ -0,0 +1,247 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const RuntimeClassManager = require('../data/managers/runtime-class-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const FogManager = require('../data/managers/iofog-manager') +const ChangeTrackingService = require('./change-tracking-service') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') +const logger = require('../logger') + +const CATALOG_HANDLERS = new Set([ + 'spin', + 'edgelet-wasmtime', + 'wasmtime', + 'wasmedge', + 'nvidia-cdi' +]) + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function toWireRuntimeClass (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + return { + name: plain.name, + handler: plain.handler + } +} + +function _warnIfUnknownHandler (name, handler) { + if (!handler || CATALOG_HANDLERS.has(handler)) { + return + } + logger.warn( + `RuntimeClass '${name}' handler '${handler}' is not in the known catalog; applying it may restart the data plane` + ) +} + +async function findLinkedFogUuids (runtimeClass, transaction) { + if (!runtimeClass || typeof runtimeClass.getFogs !== 'function') { + return [] + } + const fogs = await runtimeClass.getFogs({ transaction }) + return fogs.map((fog) => fog.uuid) +} + +async function _updateChangeTrackingForFogs (fogUuids, transaction) { + for (const fogUuid of fogUuids) { + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.runtimeClasses, transaction) + } +} + +async function findMicroservicesPinningRuntime (className, options, transaction) { + const where = { runtime: className } + if (options && options.fogUuid) { + where.iofogUuid = options.fogUuid + } + const microservices = await MicroserviceManager.findAll(where, transaction) + return microservices.map((microservice) => microservice.uuid) +} + +async function getRuntimeClassByName (name, transaction) { + const runtimeClass = await RuntimeClassManager.findOne({ name }, transaction) + if (!runtimeClass) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_NOT_FOUND, name)) + } + return runtimeClass +} + +async function listRuntimeClassesEndpoint (transaction) { + const runtimeClasses = await RuntimeClassManager.findAll({}, transaction) + return { + runtimeClasses: runtimeClasses.map(toWireRuntimeClass) + } +} + +async function getRuntimeClassEndpoint (name, transaction) { + const runtimeClass = await getRuntimeClassByName(name, transaction) + return toWireRuntimeClass(runtimeClass) +} + +async function upsertRuntimeClassEndpoint (name, data, transaction) { + const existing = await RuntimeClassManager.findOne({ name }, transaction) + if (existing) { + return updateRuntimeClassEndpoint(name, data, transaction) + } + return createRuntimeClassEndpoint({ ...data, name }, transaction) +} + +async function createRuntimeClassEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.runtimeClassCreate) + + const existing = await RuntimeClassManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_ALREADY_EXISTS, data.name)) + } + + const created = await RuntimeClassManager.create({ + name: data.name, + handler: data.handler + }, transaction) + + return toWireRuntimeClass(created) +} + +async function updateRuntimeClassEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.runtimeClassUpdate) + const runtimeClass = await getRuntimeClassByName(name, transaction) + + const nextHandler = data.handler !== undefined ? data.handler : runtimeClass.handler + const linkedFogUuids = await findLinkedFogUuids(runtimeClass, transaction) + + await RuntimeClassManager.update({ name }, { + handler: nextHandler + }, transaction) + + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + + return getRuntimeClassEndpoint(name, transaction) +} + +async function deleteRuntimeClassEndpoint (name, transaction) { + const runtimeClass = await getRuntimeClassByName(name, transaction) + const blocking = await findMicroservicesPinningRuntime(name, {}, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_IN_USE, name, blocking.join(', '))) + } + + const linkedFogUuids = await findLinkedFogUuids(runtimeClass, transaction) + await RuntimeClassManager.delete({ name }, transaction) + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + return {} +} + +async function getRuntimeClassLinkEndpoint (name, transaction) { + const runtimeClass = await getRuntimeClassByName(name, transaction) + return { + fogUuids: await findLinkedFogUuids(runtimeClass, transaction) + } +} + +async function linkRuntimeClassEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.runtimeClassLink) + const runtimeClass = await getRuntimeClassByName(name, transaction) + const alreadyLinked = new Set(await findLinkedFogUuids(runtimeClass, transaction)) + + _warnIfUnknownHandler(name, runtimeClass.handler) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + if (agent.containerEngine !== 'edgelet') { + throw new Errors.ValidationError( + AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_LINK_REQUIRES_EDGELET, fogUuid) + ) + } + await runtimeClass.addFog(agent, { transaction }) + } + + const newlyLinked = fogUuids.filter((uuid) => !alreadyLinked.has(uuid)) + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs(newlyLinked, transaction) + } + + return getRuntimeClassEndpoint(name, transaction) +} + +async function unlinkRuntimeClassEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.runtimeClassUnlink) + const runtimeClass = await getRuntimeClassByName(name, transaction) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + const blocking = await findMicroservicesPinningRuntime(name, { fogUuid }, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError( + AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_UNLINK_IN_USE, name, fogUuid, blocking.join(', ')) + ) + } + await runtimeClass.removeFog(agent, { transaction }) + } + + await _updateChangeTrackingForFogs(fogUuids, transaction) + return {} +} + +async function ensureRuntimeClassLinkedToFog (fogUuid, className, transaction) { + if (typeof className !== 'string' || className.length === 0) { + return false + } + + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + + const runtimeClass = await RuntimeClassManager.findOne({ name: className }, transaction) + if (!runtimeClass) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_MISSING, className)) + } + + if (agent.containerEngine !== 'edgelet') { + return false + } + + const alreadyLinked = typeof runtimeClass.hasFog === 'function' + ? await runtimeClass.hasFog(agent, { transaction }) + : false + if (alreadyLinked) { + return false + } + + await runtimeClass.addFog(agent, { transaction }) + await _updateChangeTrackingForFogs([fogUuid], transaction) + return true +} + +module.exports = { + listRuntimeClassesEndpoint: TransactionDecorator.generateTransaction(listRuntimeClassesEndpoint), + getRuntimeClassEndpoint: TransactionDecorator.generateTransaction(getRuntimeClassEndpoint), + createRuntimeClassEndpoint: TransactionDecorator.generateTransaction(createRuntimeClassEndpoint), + updateRuntimeClassEndpoint: TransactionDecorator.generateTransaction(updateRuntimeClassEndpoint), + deleteRuntimeClassEndpoint: TransactionDecorator.generateTransaction(deleteRuntimeClassEndpoint), + getRuntimeClassLinkEndpoint: TransactionDecorator.generateTransaction(getRuntimeClassLinkEndpoint), + linkRuntimeClassEndpoint: TransactionDecorator.generateTransaction(linkRuntimeClassEndpoint), + unlinkRuntimeClassEndpoint: TransactionDecorator.generateTransaction(unlinkRuntimeClassEndpoint), + upsertRuntimeClassEndpoint: TransactionDecorator.generateTransaction(upsertRuntimeClassEndpoint), + ensureRuntimeClassLinkedToFog: TransactionDecorator.generateTransaction(ensureRuntimeClassLinkedToFog), + toWireRuntimeClass +} diff --git a/test/src/routes/runtimeClasses-routes.test.js b/test/src/routes/runtimeClasses-routes.test.js new file mode 100644 index 00000000..1d793aac --- /dev/null +++ b/test/src/routes/runtimeClasses-routes.test.js @@ -0,0 +1,19 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('RuntimeClass routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/runtimeClasses.js'), 'utf8') + + it('registers user runtime class CRUD, YAML, and link routes', () => { + expect(source).to.include("path: '/api/v3/runtimeClasses'") + expect(source).to.include("path: '/api/v3/runtimeClasses/:name'") + expect(source).to.include("path: '/api/v3/runtimeClasses/yaml'") + expect(source).to.include("path: '/api/v3/runtimeClasses/yaml/:name'") + expect(source).to.include("path: '/api/v3/runtimeClasses/:name/link'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + }) +}) diff --git a/test/src/services/runtime-class-service.test.js b/test/src/services/runtime-class-service.test.js new file mode 100644 index 00000000..3f9fa15b --- /dev/null +++ b/test/src/services/runtime-class-service.test.js @@ -0,0 +1,285 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const RuntimeClassManager = require('../../../src/data/managers/runtime-class-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const RuntimeClassService = require('../../../src/services/runtime-class-service') +const logger = require('../../../src/logger') +const Errors = require('../../../src/helpers/errors') + +describe('RuntimeClass Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + const fogUuid = 'fog-1' + + afterEach(() => { + $sandbox.restore() + }) + + function stubRuntimeClassRow (overrides = {}) { + return { + name: 'spin', + handler: 'spin', + getFogs: $sandbox.stub().resolves([]), + addFog: $sandbox.stub().resolves(), + removeFog: $sandbox.stub().resolves(), + hasFog: $sandbox.stub().resolves(false), + ...overrides + } + } + + describe('.createRuntimeClassEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(null) + $sandbox.stub(RuntimeClassManager, 'create').callsFake(async (data) => data) + }) + + it('creates a runtime class by name', async () => { + const result = await RuntimeClassService.createRuntimeClassEndpoint({ + name: 'spin', + handler: 'spin' + }, transaction) + + expect(result).to.eql({ name: 'spin', handler: 'spin' }) + expect(RuntimeClassManager.create).to.have.been.calledOnce + }) + + it('rejects a duplicate name', async () => { + RuntimeClassManager.findOne.resolves(stubRuntimeClassRow()) + + await expect(RuntimeClassService.createRuntimeClassEndpoint({ + name: 'spin', + handler: 'spin' + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + }) + + describe('.listRuntimeClassesEndpoint() and .getRuntimeClassEndpoint()', () => { + const row = { name: 'spin', handler: 'spin' } + + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findAll').resolves([row]) + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(row) + }) + + it('lists all runtime classes', async () => { + const result = await RuntimeClassService.listRuntimeClassesEndpoint(transaction) + expect(result).to.eql({ runtimeClasses: [row] }) + }) + + it('returns spec only without linked fogs', async () => { + const result = await RuntimeClassService.getRuntimeClassEndpoint('spin', transaction) + expect(result).to.eql(row) + expect(result).to.not.have.property('fogUuids') + expect(result).to.not.have.property('fogs') + }) + + it('throws when the runtime class is missing', async () => { + RuntimeClassManager.findOne.resolves(null) + await expect(RuntimeClassService.getRuntimeClassEndpoint('missing', transaction)) + .to.be.rejectedWith(Errors.NotFoundError, /missing/) + }) + }) + + describe('.updateRuntimeClassEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(stubRuntimeClassRow()) + $sandbox.stub(RuntimeClassManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('updates handler and notifies linked agents', async () => { + RuntimeClassManager.findOne.resolves(stubRuntimeClassRow({ + getFogs: $sandbox.stub().resolves([{ uuid: fogUuid }]) + })) + + await RuntimeClassService.updateRuntimeClassEndpoint('spin', { + handler: 'wasmtime' + }, transaction) + + expect(RuntimeClassManager.update).to.have.been.calledWith( + { name: 'spin' }, + sinon.match({ handler: 'wasmtime' }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.runtimeClasses, + transaction + ) + }) + }) + + describe('.deleteRuntimeClassEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(stubRuntimeClassRow()) + $sandbox.stub(RuntimeClassManager, 'delete').resolves() + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes when no microservice pins the runtime name', async () => { + const result = await RuntimeClassService.deleteRuntimeClassEndpoint('spin', transaction) + expect(result).to.eql({}) + expect(RuntimeClassManager.delete).to.have.been.calledWith({ name: 'spin' }, transaction) + }) + + it('refuses delete when any microservice pins the runtime name', async () => { + MicroserviceManager.findAll.resolves([ + { uuid: 'ms-1', runtime: 'spin' } + ]) + + await expect(RuntimeClassService.deleteRuntimeClassEndpoint('spin', transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-1/) + expect(RuntimeClassManager.delete).to.not.have.been.called + }) + }) + + describe('link and unlink', () => { + let runtimeClass + let agent + + beforeEach(() => { + runtimeClass = stubRuntimeClassRow() + agent = { + uuid: fogUuid, + containerEngine: 'edgelet' + } + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(runtimeClass) + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + $sandbox.stub(logger, 'warn') + }) + + it('links an edgelet fog and sets the runtimeClasses change flag', async () => { + const result = await RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(runtimeClass.addFog).to.have.been.calledWith(agent, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.runtimeClasses, + transaction + ) + expect(result.name).to.equal('spin') + }) + + it('refuses link when the fog is not using edgelet', async () => { + agent.containerEngine = 'docker' + await expect(RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ValidationError, /edgelet/) + expect(runtimeClass.addFog).to.not.have.been.called + }) + + it('warns when the handler is not in the known catalog', async () => { + runtimeClass.handler = 'custom-oci' + await RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(logger.warn).to.have.been.calledOnce + }) + + it('does not set the change flag when the fog is already linked', async () => { + runtimeClass.getFogs.resolves([{ uuid: fogUuid }]) + await RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(ChangeTrackingService.update).to.not.have.been.called + }) + + it('returns linked fog uuids', async () => { + runtimeClass.getFogs.resolves([{ uuid: fogUuid }]) + const result = await RuntimeClassService.getRuntimeClassLinkEndpoint('spin', transaction) + expect(result).to.eql({ fogUuids: [fogUuid] }) + }) + + it('unlinks a fog and sets the runtimeClasses change flag', async () => { + await RuntimeClassService.unlinkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(runtimeClass.removeFog).to.have.been.calledWith(agent, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.runtimeClasses, + transaction + ) + }) + + it('refuses unlink when a microservice on that fog pins the runtime name', async () => { + MicroserviceManager.findAll.resolves([ + { uuid: 'ms-bound', iofogUuid: fogUuid, runtime: 'spin' } + ]) + + await expect(RuntimeClassService.unlinkRuntimeClassEndpoint('spin', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-bound/) + expect(runtimeClass.removeFog).to.not.have.been.called + }) + }) + + describe('.upsertRuntimeClassEndpoint()', () => { + it('creates when the name is new and updates when it already exists', async () => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(null) + $sandbox.stub(RuntimeClassManager, 'create').callsFake(async (data) => data) + + const created = await RuntimeClassService.upsertRuntimeClassEndpoint('spin', { + handler: 'spin' + }, transaction) + expect(created.name).to.equal('spin') + expect(RuntimeClassManager.create).to.have.been.calledOnce + + const existing = stubRuntimeClassRow({ + getFogs: $sandbox.stub().resolves([]) + }) + RuntimeClassManager.findOne.resolves(existing) + $sandbox.stub(RuntimeClassManager, 'update').resolves() + + await RuntimeClassService.upsertRuntimeClassEndpoint('spin', { + handler: 'wasmtime' + }, transaction) + expect(RuntimeClassManager.update).to.have.been.calledOnce + expect(RuntimeClassManager.create).to.have.been.calledOnce + }) + }) + + describe('.ensureRuntimeClassLinkedToFog()', () => { + let runtimeClass + let agent + + beforeEach(() => { + runtimeClass = stubRuntimeClassRow() + agent = { + uuid: fogUuid, + containerEngine: 'edgelet' + } + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(runtimeClass) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links a missing class on an edgelet fog and is idempotent when already linked', async () => { + const first = await RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction) + expect(first).to.equal(true) + expect(runtimeClass.addFog).to.have.been.calledOnce + expect(ChangeTrackingService.update).to.have.been.calledOnce + + runtimeClass.hasFog.resolves(true) + ChangeTrackingService.update.resetHistory() + runtimeClass.addFog.resetHistory() + + const second = await RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction) + expect(second).to.equal(false) + expect(runtimeClass.addFog).to.not.have.been.called + expect(ChangeTrackingService.update).to.not.have.been.called + }) + + it('does not link when the fog is not using edgelet', async () => { + agent.containerEngine = 'docker' + const linked = await RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction) + expect(linked).to.equal(false) + expect(runtimeClass.addFog).to.not.have.been.called + }) + + it('rejects when the runtime class row is missing', async () => { + RuntimeClassManager.findOne.resolves(null) + await expect(RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction)) + .to.be.rejectedWith(Errors.ValidationError, /does not exist/) + }) + }) +}) From c2e60b278acb217df51560488d156ef700fe30e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:50:47 +0300 Subject: [PATCH 06/55] Extend registries with type, CA, and insecure, and keep image pulls OCI-only. Hub is the Hugging Face system registry; microservice and catalog images reject hf registries. --- src/helpers/constants.js | 1 + src/schemas/registry.js | 25 ++- src/services/registry-service.js | 212 ++++++++++++++++++++- test/src/services/registry-service.test.js | 181 +++++++++++++++++- 4 files changed, 394 insertions(+), 25 deletions(-) diff --git a/src/helpers/constants.js b/src/helpers/constants.js index 694dfa92..1e0c0441 100644 --- a/src/helpers/constants.js +++ b/src/helpers/constants.js @@ -55,6 +55,7 @@ module.exports = { HTTP_CODE_NOT_IMPLEMENTED: 501, READINESS_RETRY_AFTER_SECONDS: 5, HTTP_CODE_DUPLICATE_PROPERTY: 409, + HTTP_CODE_CONFLICT: 409, HTTP_CODE_INTERNAL_ERROR: 500, DEFAULT_ROUTER_NAME: 'default-router', diff --git a/src/schemas/registry.js b/src/schemas/registry.js index e07a46d7..2b19b76f 100644 --- a/src/schemas/registry.js +++ b/src/schemas/registry.js @@ -1,18 +1,23 @@ +const emailPattern = '^$|^(([^<>()\\[\\]\\\\.,;:\\s@"]+(\\.[^<>()\\[\\]\\\\.,;:\\s@"]+)*)|(".+"))@((\\[[0-9]{1,3}\\.[0-9]{1,3}' + + '\\.[0-9]{1,3}\\.[0-9]{1,3}])|(([a-zA-Z\\-0-9]+\\.)+[a-zA-Z]{2,}))$' + const registryCreate = { id: '/registryCreate', type: 'object', properties: { url: { type: 'string', minLength: 1 }, isPublic: { type: 'boolean' }, - username: { type: 'string', minLength: 1 }, + username: { type: 'string' }, password: { type: 'string' }, email: { type: 'string', - pattern: '^(([^<>()\\[\\]\\\\.,;:\\s@"]+(\\.[^<>()\\[\\]\\\\.,;:\\s@"]+)*)|(".+"))@((\\[[0-9]{1,3}\\.[0-9]{1,3}' + - '\\.[0-9]{1,3}\\.[0-9]{1,3}])|(([a-zA-Z\\-0-9]+\\.)+[a-zA-Z]{2,}))$' - } + pattern: emailPattern + }, + type: { type: 'string', enum: ['oci', 'hf'] }, + ca: { type: 'string' }, + insecure: { type: 'boolean' } }, - required: ['url', 'isPublic', 'username', 'password', 'email'], + required: ['url', 'isPublic'], additionalProperties: true } @@ -32,13 +37,15 @@ const registryUpdate = { properties: { url: { type: 'string', minLength: 1 }, isPublic: { type: 'boolean' }, - username: { type: 'string', minLength: 1 }, + username: { type: 'string' }, password: { type: 'string' }, email: { type: 'string', - pattern: '^(([^<>()\\[\\]\\\\.,;:\\s@"]+(\\.[^<>()\\[\\]\\\\.,;:\\s@"]+)*)|(".+"))@((\\[[0-9]{1,3}\\.[0-9]{1,3}' + - '\\.[0-9]{1,3}\\.[0-9]{1,3}])|(([a-zA-Z\\-0-9]+\\.)+[a-zA-Z]{2,}))$' - } + pattern: emailPattern + }, + type: { type: 'string', enum: ['oci', 'hf'] }, + ca: { type: 'string' }, + insecure: { type: 'boolean' } }, additionalProperties: true } diff --git a/src/services/registry-service.js b/src/services/registry-service.js index 4f94b0ba..a82fd1c7 100644 --- a/src/services/registry-service.js +++ b/src/services/registry-service.js @@ -10,26 +10,180 @@ const ErrorMessages = require('../helpers/error-messages') const ReconcileOutboxManager = require('../data/managers/reconcile-outbox-manager') const TransactionDecorator = require('../decorators/transaction-decorator') const MicroserviceManager = require('../data/managers/microservice-manager') -// const Sequelize = require('sequelize') -// const Op = Sequelize.Op const AppHelper = require('../helpers/app-helper') +const REGISTRY_TYPE_OCI = 'oci' +const REGISTRY_TYPE_HF = 'hf' +const HUB_REGISTRY_URL = 'https://huggingface.co' +const HUB_REGISTRY_TYPE = REGISTRY_TYPE_HF +const LEGACY_SYSTEM_REGISTRY_IDS = new Set([1, 2]) + function isPasswordEmpty (password) { return password == null || (typeof password === 'string' && password.trim() === '') } +function isBlank (value) { + return value == null || (typeof value === 'string' && value.trim() === '') +} + +function normalizeRegistryType (type) { + if (type == null || type === '') { + return REGISTRY_TYPE_OCI + } + return type +} + +function isHubRegistry (registry) { + if (!registry) { + return false + } + return registry.url === HUB_REGISTRY_URL && normalizeRegistryType(registry.type) === HUB_REGISTRY_TYPE +} + +function isLegacySystemRegistryId (id) { + return LEGACY_SYSTEM_REGISTRY_IDS.has(parseInt(id, 10)) +} + +function toPlainRegistry (registry) { + if (!registry) { + return null + } + return typeof registry.toJSON === 'function' ? registry.toJSON() : { ...registry } +} + +function toAgentRegistry (registry) { + const plain = toPlainRegistry(registry) || {} + return { + id: plain.id, + url: plain.url, + isPublic: !!plain.isPublic, + username: plain.username || '', + password: plain.password || '', + userEmail: plain.userEmail || '', + type: normalizeRegistryType(plain.type), + ca: plain.ca || '', + insecure: !!plain.insecure + } +} + +function isValidBase64 (value) { + if (value == null || value === '') { + return true + } + if (typeof value !== 'string') { + return false + } + const normalized = value.replace(/\s/g, '') + if (!normalized) { + return true + } + if (normalized.length % 4 !== 0) { + return false + } + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(normalized)) { + return false + } + try { + Buffer.from(normalized, 'base64') + return true + } catch (err) { + return false + } +} + +function normalizeCaForStorage (ca) { + if (ca === undefined || ca === null || (typeof ca === 'string' && ca.trim() === '')) { + return null + } + if (!isValidBase64(ca)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_INVALID_CA) + } + return ca +} + +function normalizeCaForWire (ca) { + if (ca == null || (typeof ca === 'string' && ca.trim() === '')) { + return null + } + return ca +} + +function assertCredentialRules (data, existing) { + const type = normalizeRegistryType(data.type !== undefined ? data.type : existing && existing.type) + const isPublic = data.isPublic !== undefined ? data.isPublic : !!(existing && existing.isPublic) + const username = data.username !== undefined ? data.username : (existing && existing.username) + const password = data.password !== undefined ? data.password : (existing && existing.password) + + if (isPublic) { + return + } + if (type === REGISTRY_TYPE_OCI) { + if (isBlank(username) || isPasswordEmpty(password)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_PRIVATE_OCI_CREDENTIALS_REQUIRED) + } + return + } + if (type === REGISTRY_TYPE_HF && isPasswordEmpty(password)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_PRIVATE_HF_TOKEN_REQUIRED) + } +} + +function assertSystemRegistryProtection (existing, patch) { + if (isLegacySystemRegistryId(existing.id)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + if (!isHubRegistry(existing)) { + return + } + if (patch.url !== undefined && patch.url !== existing.url) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + if (patch.type !== undefined && normalizeRegistryType(patch.type) !== HUB_REGISTRY_TYPE) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } +} + +function mergedRegistryIdentity (existing, patch) { + return { + url: patch.url !== undefined ? patch.url : existing && existing.url, + type: normalizeRegistryType(patch.type !== undefined ? patch.type : existing && existing.type) + } +} + const createRegistry = async function (registry, transaction) { await Validator.validate(registry, Validator.schemas.registryCreate) + const type = normalizeRegistryType(registry.type) + const ca = normalizeCaForStorage(registry.ca) + assertCredentialRules({ + type, + isPublic: registry.isPublic, + username: registry.username, + password: registry.password + }, null) + + if (isHubRegistry({ url: registry.url, type })) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + let registryCreate = { url: registry.url, username: registry.username, password: registry.password, isPublic: registry.isPublic, - userEmail: registry.email + userEmail: registry.email, + type, + ca, + insecure: registry.insecure === true } registryCreate = AppHelper.deleteUndefinedFields(registryCreate) + if (registryCreate.type === undefined) { + registryCreate.type = REGISTRY_TYPE_OCI + } + if (registryCreate.insecure === undefined) { + registryCreate.insecure = false + } const createdRegistry = await RegistryManager.create(registryCreate, transaction) @@ -70,7 +224,15 @@ const findRegistries = async function (isCLI, transaction) { const registries = await RegistryManager.findAllWithAttributes(queryRegistry, { exclude: ['password'] }, transaction) return { - registries + registries: registries.map((registry) => { + const plain = toPlainRegistry(registry) + return { + ...plain, + type: normalizeRegistryType(plain.type), + ca: normalizeCaForWire(plain.ca), + insecure: !!plain.insecure + } + }) } } @@ -79,15 +241,17 @@ const deleteRegistry = async function (registryData, isCLI, transaction) { const queryData = isCLI ? { id: registryData.id } : { id: registryData.id } - // Convert registryId to number to handle string IDs from URL parameters const id = parseInt(registryData.id, 10) - if (id === 1 || id === 2) { + if (isLegacySystemRegistryId(id)) { throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) } const registry = await RegistryManager.findOne(queryData, transaction) if (!registry) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryData.id)) } + if (isHubRegistry(registry)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } const microservices = await MicroserviceManager.findAllWithStatuses({ registryId: registryData.id }, transaction) if (microservices.length > 0) { throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_IN_USE) @@ -103,9 +267,8 @@ const deleteRegistry = async function (registryData, isCLI, transaction) { const updateRegistry = async function (registry, registryId, isCLI, transaction) { await Validator.validate(registry, Validator.schemas.registryUpdate) - // Convert registryId to number to handle string IDs from URL parameters const id = parseInt(registryId, 10) - if (id === 1 || id === 2) { + if (isLegacySystemRegistryId(id)) { throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) } const existingRegistry = await RegistryManager.findOne({ @@ -116,12 +279,24 @@ const updateRegistry = async function (registry, registryId, isCLI, transaction) throw new Errors.NotFoundError(ErrorMessages.REGISTRY_NOT_FOUND) } + assertSystemRegistryProtection(existingRegistry, registry) + + const mergedIdentity = mergedRegistryIdentity(existingRegistry, registry) + if (!isHubRegistry(existingRegistry) && isHubRegistry(mergedIdentity)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + + assertCredentialRules(registry, existingRegistry) + let registryUpdate = { url: registry.url, username: registry.username, password: registry.password, isPublic: registry.isPublic, - userEmail: registry.email + userEmail: registry.email, + type: registry.type, + ca: registry.ca !== undefined ? normalizeCaForStorage(registry.ca) : undefined, + insecure: registry.insecure } registryUpdate = AppHelper.deleteUndefinedFields(registryUpdate) @@ -160,6 +335,21 @@ const getRegistry = async function (registryId, isCLI, transaction) { if (!registry) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryId)) } + const plain = toPlainRegistry(registry) + return { + ...plain, + type: normalizeRegistryType(plain.type), + ca: normalizeCaForWire(plain.ca), + insecure: !!plain.insecure + } +} + +const assertOciRegistryForImage = async function (registryId, transaction) { + const id = parseInt(registryId, 10) + const registry = Number.isFinite(id) ? await RegistryManager.findOne({ id }, transaction) : null + if (!registry || normalizeRegistryType(registry.type) !== REGISTRY_TYPE_OCI) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, registryId)) + } return registry } @@ -168,5 +358,7 @@ module.exports = { findRegistries: TransactionDecorator.generateTransaction(findRegistries), deleteRegistry: TransactionDecorator.generateTransaction(deleteRegistry), updateRegistry: TransactionDecorator.generateTransaction(updateRegistry), - getRegistry: TransactionDecorator.generateTransaction(getRegistry) + getRegistry: TransactionDecorator.generateTransaction(getRegistry), + assertOciRegistryForImage: TransactionDecorator.generateTransaction(assertOciRegistryForImage), + toAgentRegistry } diff --git a/test/src/services/registry-service.test.js b/test/src/services/registry-service.test.js index dae3da1a..65fadb36 100644 --- a/test/src/services/registry-service.test.js +++ b/test/src/services/registry-service.test.js @@ -23,6 +23,9 @@ function buildRegistryRecord (fields = {}) { password: 'encrypted-secret', isPublic: false, userEmail: 'user@example.com', + type: 'oci', + ca: null, + insecure: false, ...fields } } @@ -65,7 +68,10 @@ describe('Registry Service', () => { expect(RegistryManager.create).to.have.been.calledWithMatch({ url: registryData.url, username: registryData.username, - userEmail: registryData.email + userEmail: registryData.email, + type: 'oci', + ca: null, + insecure: false }, transaction) expect(SecretHelper.encryptSecretInternal).to.have.been.calledWith( { value: registryData.password }, @@ -100,6 +106,75 @@ describe('Registry Service', () => { expect(RegistryManager.update).to.not.have.been.called }) }) + + it('rejects private OCI registries without username and password', () => { + return expect($service.createRegistry({ + url: 'https://registry.example.com', + isPublic: false, + type: 'oci' + }, transaction)).to.be.rejectedWith(ErrorMessages.REGISTRY_PRIVATE_OCI_CREDENTIALS_REQUIRED) + }) + + it('creates a private Hugging Face registry with a token', async () => { + const hfRegistry = { + url: 'https://hf.example.com', + isPublic: false, + type: 'hf', + password: 'hf_token' + } + RegistryManager.create.resolves(buildRegistryRecord({ + id: 18, + url: hfRegistry.url, + type: 'hf', + password: hfRegistry.password, + isPublic: false, + username: '' + })) + + const result = await $service.createRegistry(hfRegistry, transaction) + + expect(RegistryManager.create).to.have.been.calledWithMatch({ + url: hfRegistry.url, + type: 'hf', + password: hfRegistry.password, + insecure: false + }, transaction) + expect(result).to.eql({ id: 18 }) + }) + + it('rejects creating a second Hub registry', () => { + return expect($service.createRegistry({ + url: 'https://huggingface.co', + isPublic: true, + type: 'hf' + }, transaction)).to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + }) + + it('rejects an invalid CA bundle', () => { + return expect($service.createRegistry({ + url: 'https://registry.example.com', + isPublic: true, + ca: 'not-valid-base64!!!' + }, transaction)).to.be.rejectedWith(ErrorMessages.REGISTRY_INVALID_CA) + }) + + it('stores empty ca as null', async () => { + await $service.createRegistry({ + url: 'https://registry.example.com', + isPublic: true, + ca: '' + }, transaction) + expect(RegistryManager.create).to.have.been.calledWithMatch({ ca: null }, transaction) + }) + + it('stores a valid base64 ca bundle', async () => { + await $service.createRegistry({ + url: 'https://registry.example.com', + isPublic: true, + ca: 'Y2VydA==' + }, transaction) + expect(RegistryManager.create).to.have.been.calledWithMatch({ ca: 'Y2VydA==' }, transaction) + }) }) describe('.findRegistries()', () => { @@ -111,14 +186,24 @@ describe('Registry Service', () => { $sandbox.stub(RegistryManager, 'findAllWithAttributes').resolves(registries) }) - it('returns registries without password field', async () => { + it('returns registries without password field and with type, ca, and insecure defaults', async () => { const result = await $subject expect(RegistryManager.findAllWithAttributes).to.have.been.calledWith( {}, { exclude: ['password'] }, transaction ) - expect(result.registries).to.equal(registries) + expect(result.registries).to.eql([{ + id: 16, + url: 'https://registry.example.com', + username: 'user', + password: 'encrypted-secret', + isPublic: false, + userEmail: 'user@example.com', + type: 'oci', + ca: null, + insecure: false + }]) }) }) @@ -151,6 +236,19 @@ describe('Registry Service', () => { .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) }) + it('rejects the Hub registry by url and type regardless of id', () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 9, + url: 'https://huggingface.co', + type: 'hf', + isPublic: true, + username: '', + password: '' + })) + return expect($service.deleteRegistry({ id: 9 }, isCLI, transaction)) + .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + }) + context('when registry is missing', () => { beforeEach(() => { RegistryManager.findOne.resolves(null) @@ -209,6 +307,44 @@ describe('Registry Service', () => { .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) }) + it('rejects Hub url or type changes that would break identity', async () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 9, + url: 'https://huggingface.co', + type: 'hf', + isPublic: true + })) + await expect($service.updateRegistry({ url: 'https://example.com' }, 9, isCLI, transaction)) + .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + await expect($service.updateRegistry({ type: 'oci' }, 9, isCLI, transaction)) + .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + }) + + it('clears ca to null when an empty string is provided', async () => { + await $service.updateRegistry({ ca: '' }, registryId, isCLI, transaction) + expect(RegistryManager.update).to.have.been.calledWith( + { id: registryId }, + sinon.match({ ca: null }), + transaction + ) + }) + + it('allows Hub token updates without changing identity fields', async () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 9, + url: 'https://huggingface.co', + type: 'hf', + isPublic: true, + password: '' + })) + await $service.updateRegistry({ password: 'hf_token' }, 9, isCLI, transaction) + expect(RegistryManager.update).to.have.been.calledWith( + { id: 9 }, + sinon.match({ password: 'hf_token' }), + transaction + ) + }) + context('when registry is missing', () => { beforeEach(() => { RegistryManager.findOne.resolves(null) @@ -243,7 +379,7 @@ describe('Registry Service', () => { context('when password is cleared and vault reference exists', () => { beforeEach(() => { $sandbox.stub(vaultManager, 'isEnabled').returns(true) - RegistryManager.findOne.resolves({ ...existing, password: 'vault:ref' }) + RegistryManager.findOne.resolves({ ...existing, password: 'vault:ref', isPublic: true }) $sandbox.stub(SecretHelper, 'isVaultReference').returns(true) $sandbox.stub(SecretHelper, 'deleteSecret').resolves() }) @@ -268,10 +404,15 @@ describe('Registry Service', () => { $sandbox.stub(RegistryManager, 'findOne').resolves(registry) }) - it('returns the registry record', async () => { + it('returns the registry record with type, ca, and insecure', async () => { const result = await $subject expect(RegistryManager.findOne).to.have.been.calledWith({ id: registryId }, transaction) - expect(result).to.equal(registry) + expect(result).to.include({ + id: registryId, + type: 'oci', + ca: null, + insecure: false + }) }) context('when registry is missing', () => { @@ -282,4 +423,32 @@ describe('Registry Service', () => { it('rejects with NotFoundError', () => expect($subject).to.be.rejectedWith(Errors.NotFoundError)) }) }) + + describe('.assertOciRegistryForImage()', () => { + beforeEach(() => { + $sandbox.stub(RegistryManager, 'findOne') + }) + + it('accepts an OCI registry', async () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ id: 4, type: 'oci' })) + const registry = await $service.assertOciRegistryForImage(4, transaction) + expect(registry.type).to.equal('oci') + }) + + it('rejects a Hugging Face registry id', () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 3, + url: 'https://huggingface.co', + type: 'hf' + })) + return expect($service.assertOciRegistryForImage(3, transaction)) + .to.be.rejectedWith(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 3)) + }) + + it('rejects a missing registry', () => { + RegistryManager.findOne.resolves(null) + return expect($service.assertOciRegistryForImage(99, transaction)) + .to.be.rejectedWith(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 99)) + }) + }) }) From c6f472c3b1619a088d80947a4fc41467ac50ec6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:50:54 +0300 Subject: [PATCH 07/55] Add microservice catalog PATCH and persist Edgelet container fields. Catalog-only edits use a dedicated change flag and rebuild only when bind path, permissions, or empty/non-empty catalog transitions require it. --- src/controllers/microservices-controller.js | 7 + src/data/managers/application-manager.js | 30 +- .../managers/microservice-device-manager.js | 22 + .../microservice-entrypoint-manager.js | 22 + src/data/managers/microservice-manager.js | 50 ++ .../microservice-model-item-manager.js | 12 + .../managers/microservice-model-manager.js | 12 + .../managers/microservice-tmpfs-manager.js | 22 + .../managers/microservice-ulimit-manager.js | 22 + src/helpers/microservice-container-catalog.js | 632 ++++++++++++++++++ src/routes/microservices.js | 36 + src/schemas/microservice-container-spec.js | 64 ++ src/schemas/microservice.js | 214 +++++- src/services/event-service.js | 3 + src/services/microservice-workload-spec.js | 117 +++- src/services/microservices-service.js | 376 ++++++++++- .../microservice-container-catalog.test.js | 174 +++++ test/src/routes/microservices-routes.test.js | 12 + .../services/microservices-service.test.js | 517 +++++++++++++- 19 files changed, 2310 insertions(+), 34 deletions(-) create mode 100644 src/data/managers/microservice-device-manager.js create mode 100644 src/data/managers/microservice-entrypoint-manager.js create mode 100644 src/data/managers/microservice-model-item-manager.js create mode 100644 src/data/managers/microservice-model-manager.js create mode 100644 src/data/managers/microservice-tmpfs-manager.js create mode 100644 src/data/managers/microservice-ulimit-manager.js create mode 100644 src/helpers/microservice-container-catalog.js create mode 100644 test/src/helpers/microservice-container-catalog.test.js create mode 100644 test/src/routes/microservices-routes.test.js diff --git a/src/controllers/microservices-controller.js b/src/controllers/microservices-controller.js index ab737c29..704596cd 100644 --- a/src/controllers/microservices-controller.js +++ b/src/controllers/microservices-controller.js @@ -62,6 +62,12 @@ const updateSystemMicroserviceYAMLEndPoint = async function (req) { return MicroservicesService.updateSystemMicroserviceEndPoint(microserviceUuid, microservice, false) } +const updateMicroserviceCatalogEndPoint = async function (req) { + const microserviceUuid = req.params.uuid + const catalog = req.body + return MicroservicesService.updateMicroserviceCatalogEndPoint(microserviceUuid, catalog, false) +} + const updateMicroserviceConfigEndPoint = async function (req) { const microserviceUuid = req.params.uuid const config = req.body @@ -214,6 +220,7 @@ module.exports = { createMicroserviceYAMLEndPoint, updateMicroserviceYAMLEndPoint, updateSystemMicroserviceYAMLEndPoint, + updateMicroserviceCatalogEndPoint, updateMicroserviceConfigEndPoint, getMicroserviceConfigEndPoint, updateSystemMicroserviceConfigEndPoint, diff --git a/src/data/managers/application-manager.js b/src/data/managers/application-manager.js index 38c9d7cc..9720e5f0 100644 --- a/src/data/managers/application-manager.js +++ b/src/data/managers/application-manager.js @@ -2,6 +2,20 @@ const BaseManager = require('./base-manager') const models = require('../models') const Application = models.Application const Microservice = models.Microservice +const RbacServiceAccount = models.RbacServiceAccount + +const microserviceInclude = { + model: Microservice, + as: 'microservices', + required: false, + include: [ + { + model: RbacServiceAccount, + as: 'serviceAccount', + required: false + } + ] +} class ApplicationManager extends BaseManager { getEntity () { @@ -44,13 +58,7 @@ class ApplicationManager extends BaseManager { async findOnePopulated (where, attributes, transaction) { const application = await Application.findOne({ - include: [ - { - model: Microservice, - as: 'microservices', - required: false - } - ], + include: [microserviceInclude], where, attributes }, { transaction }) @@ -66,13 +74,7 @@ class ApplicationManager extends BaseManager { async findAllPopulated (where, attributes, transaction) { const applications = await Application.findAll({ - include: [ - { - model: Microservice, - as: 'microservices', - required: false - } - ], + include: [microserviceInclude], where, attributes }, { transaction }) diff --git a/src/data/managers/microservice-device-manager.js b/src/data/managers/microservice-device-manager.js new file mode 100644 index 00000000..bd0c7705 --- /dev/null +++ b/src/data/managers/microservice-device-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceDevice = models.MicroserviceDevice + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceDeviceManager extends BaseManager { + getEntity () { + return MicroserviceDevice + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceDeviceManager() +module.exports = instance diff --git a/src/data/managers/microservice-entrypoint-manager.js b/src/data/managers/microservice-entrypoint-manager.js new file mode 100644 index 00000000..90f3fca3 --- /dev/null +++ b/src/data/managers/microservice-entrypoint-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceEntrypoint = models.MicroserviceEntrypoint + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceEntrypointManager extends BaseManager { + getEntity () { + return MicroserviceEntrypoint + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceEntrypointManager() +module.exports = instance diff --git a/src/data/managers/microservice-manager.js b/src/data/managers/microservice-manager.js index ea2f9618..d643b434 100644 --- a/src/data/managers/microservice-manager.js +++ b/src/data/managers/microservice-manager.js @@ -9,6 +9,12 @@ const MicroserviceCdiDev = models.MicroserviceCdiDev const MicroserviceCapAdd = models.MicroserviceCapAdd const MicroserviceCapDrop = models.MicroserviceCapDrop const VolumeMapping = models.VolumeMapping +const MicroserviceEntrypoint = models.MicroserviceEntrypoint +const MicroserviceDevice = models.MicroserviceDevice +const MicroserviceTmpfs = models.MicroserviceTmpfs +const MicroserviceUlimit = models.MicroserviceUlimit +const MicroserviceModel = models.MicroserviceModel +const MicroserviceModelItem = models.MicroserviceModelItem const CatalogItem = models.CatalogItem const CatalogItemImage = models.CatalogItemImage const Fog = models.Fog @@ -31,6 +37,47 @@ const microserviceExcludedFields = [ 'iofog_uuid' ] +function containerChildIncludes () { + return [ + { + model: MicroserviceEntrypoint, + as: 'entrypoint', + required: false, + attributes: ['id', 'entrypoint'] + }, + { + model: MicroserviceDevice, + as: 'devices', + required: false, + attributes: ['id', 'hostPath', 'containerPath', 'permissions'] + }, + { + model: MicroserviceTmpfs, + as: 'tmpfs', + required: false, + attributes: ['id', 'containerPath', 'size', 'mode'] + }, + { + model: MicroserviceUlimit, + as: 'ulimits', + required: false, + attributes: ['id', 'name', 'soft', 'hard'] + }, + { + model: MicroserviceModel, + as: 'microserviceModel', + required: false, + attributes: ['bindPath', 'permissions'] + }, + { + model: MicroserviceModelItem, + as: 'modelItems', + required: false, + attributes: ['id', 'name'] + } + ] +} + class MicroserviceManager extends BaseManager { getEntity () { return Microservice @@ -56,6 +103,7 @@ class MicroserviceManager extends BaseManager { required: false, attributes: ['cmd'] }, + ...containerChildIncludes(), { model: MicroserviceCdiDev, as: 'cdiDevices', @@ -152,6 +200,7 @@ class MicroserviceManager extends BaseManager { required: false, attributes: ['cmd', 'id'] }, + ...containerChildIncludes(), { model: MicroserviceCdiDev, as: 'cdiDevices', @@ -272,6 +321,7 @@ class MicroserviceManager extends BaseManager { required: false, attributes: ['cmd'] }, + ...containerChildIncludes(), { model: MicroserviceCdiDev, as: 'cdiDevices', diff --git a/src/data/managers/microservice-model-item-manager.js b/src/data/managers/microservice-model-item-manager.js new file mode 100644 index 00000000..9d05f67e --- /dev/null +++ b/src/data/managers/microservice-model-item-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceModelItem = models.MicroserviceModelItem + +class MicroserviceModelItemManager extends BaseManager { + getEntity () { + return MicroserviceModelItem + } +} + +const instance = new MicroserviceModelItemManager() +module.exports = instance diff --git a/src/data/managers/microservice-model-manager.js b/src/data/managers/microservice-model-manager.js new file mode 100644 index 00000000..3c0f3c9a --- /dev/null +++ b/src/data/managers/microservice-model-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceModel = models.MicroserviceModel + +class MicroserviceModelManager extends BaseManager { + getEntity () { + return MicroserviceModel + } +} + +const instance = new MicroserviceModelManager() +module.exports = instance diff --git a/src/data/managers/microservice-tmpfs-manager.js b/src/data/managers/microservice-tmpfs-manager.js new file mode 100644 index 00000000..207c5569 --- /dev/null +++ b/src/data/managers/microservice-tmpfs-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceTmpfs = models.MicroserviceTmpfs + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceTmpfsManager extends BaseManager { + getEntity () { + return MicroserviceTmpfs + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceTmpfsManager() +module.exports = instance diff --git a/src/data/managers/microservice-ulimit-manager.js b/src/data/managers/microservice-ulimit-manager.js new file mode 100644 index 00000000..395c8fb1 --- /dev/null +++ b/src/data/managers/microservice-ulimit-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceUlimit = models.MicroserviceUlimit + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceUlimitManager extends BaseManager { + getEntity () { + return MicroserviceUlimit + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceUlimitManager() +module.exports = instance diff --git a/src/helpers/microservice-container-catalog.js b/src/helpers/microservice-container-catalog.js new file mode 100644 index 00000000..3a815fe1 --- /dev/null +++ b/src/helpers/microservice-container-catalog.js @@ -0,0 +1,632 @@ +'use strict' + +const Errors = require('./errors') +const ErrorMessages = require('./error-messages') +const AppHelper = require('./app-helper') +const { serviceNameRegex } = require('../schemas/utils/utils') + +const SERVICE_NAME_RE = new RegExp(`^${serviceNameRegex}$`) + +const SYSCTL_ALLOWLIST = new Set([ + 'kernel.shm_rmid_forced', + 'net.ipv4.ip_local_port_range', + 'net.ipv4.tcp_syncookies', + 'net.ipv4.ping_group_range', + 'net.ipv4.ip_unprivileged_port_start', + 'net.ipv4.ip_local_reserved_ports', + 'net.ipv4.tcp_keepalive_time', + 'net.ipv4.tcp_fin_timeout', + 'net.ipv4.tcp_keepalive_intvl', + 'net.ipv4.tcp_keepalive_probes', + 'net.ipv4.tcp_rmem', + 'net.ipv4.tcp_wmem', + 'net.ipv4.tcp_slow_start_after_idle', + 'net.ipv4.tcp_notsent_lowat' +]) + +const ULIMIT_ALLOWLIST = new Set([ + 'core', 'cpu', 'data', 'fsize', 'locks', 'memlock', 'msgqueue', 'nice', + 'nofile', 'nproc', 'rss', 'rtprio', 'rttime', 'sigpending', 'stack' +]) + +function parseJsonField (raw) { + if (raw == null || raw === '') { + return null + } + if (typeof raw === 'object') { + return raw + } + if (typeof raw !== 'string') { + return null + } + try { + return JSON.parse(raw) + } catch (error) { + return null + } +} + +function serializeJsonField (value) { + if (value == null) { + return null + } + return JSON.stringify(value) +} + +function parseArgv (raw) { + const parsed = Array.isArray(raw) ? raw : parseJsonField(raw) + if (!Array.isArray(parsed)) { + return [] + } + return parsed.filter((item) => typeof item === 'string') +} + +function serializeArgv (argv) { + if (!Array.isArray(argv) || argv.length === 0) { + return null + } + return JSON.stringify(argv) +} + +function resolveProcessArgv (data) { + if (!data || typeof data !== 'object') { + return undefined + } + if (data.commands !== undefined) { + return data.commands + } + if (data.cmd !== undefined) { + return data.cmd + } + return undefined +} + +function assertArgv (name, argv) { + if (argv === undefined || argv === null) { + return + } + if (!Array.isArray(argv) || argv.some((item) => typeof item !== 'string')) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ARGV_TYPE, name)) + } +} + +function isCatalogEmpty (catalog) { + return !catalog || !Array.isArray(catalog.items) || catalog.items.length === 0 +} + +function parseCatalog (raw) { + return parseJsonField(raw) +} + +function normalizeCatalog (input) { + if (input == null) { + return null + } + const items = Array.isArray(input.items) + ? input.items.map((item) => ({ name: item && item.name })) + : [] + const permissions = input.permissions == null || input.permissions === '' + ? 'ro' + : input.permissions + return { + bindPath: input.bindPath, + permissions, + items + } +} + +function serializeCatalog (catalog) { + if (catalog == null) { + return null + } + return JSON.stringify({ + bindPath: catalog.bindPath || '', + permissions: catalog.permissions || 'ro', + items: Array.isArray(catalog.items) ? catalog.items : [] + }) +} + +function catalogItemNames (catalog) { + if (isCatalogEmpty(catalog)) { + return [] + } + return catalog.items.map((item) => item && item.name).filter(Boolean) +} + +function catalogRequiresRebuild (previousRaw, nextCatalog) { + const previous = previousRaw && typeof previousRaw === 'object' && !Array.isArray(previousRaw) + ? previousRaw + : parseCatalog(previousRaw) + const previousEmpty = isCatalogEmpty(previous) + const nextEmpty = isCatalogEmpty(nextCatalog) + if (previousEmpty !== nextEmpty) { + return true + } + if (previousEmpty && nextEmpty) { + return false + } + const previousBind = (previous && previous.bindPath) || '' + const nextBind = (nextCatalog && nextCatalog.bindPath) || '' + const previousPermissions = (previous && previous.permissions) || 'ro' + const nextPermissions = (nextCatalog && nextCatalog.permissions) || 'ro' + return previousBind !== nextBind || previousPermissions !== nextPermissions +} + +function isAbsoluteContainerPath (value) { + return typeof value === 'string' && value.startsWith('/') && value.length > 1 +} + +function joinCatalogItemPath (bindPath, name) { + const base = (bindPath || '').replace(/\/+$/, '') + return `${base}/${name}/` +} + +function collectMountPaths (volumeMappings, tmpfs) { + const paths = [] + if (Array.isArray(volumeMappings)) { + for (const mapping of volumeMappings) { + if (mapping && mapping.containerDestination) { + paths.push(mapping.containerDestination) + } + } + } + const tmpfsList = Array.isArray(tmpfs) ? tmpfs : parseJsonField(tmpfs) + if (Array.isArray(tmpfsList)) { + for (const entry of tmpfsList) { + if (entry && entry.containerPath) { + paths.push(entry.containerPath) + } + } + } + return paths +} + +function pathsCollide (left, right) { + if (!left || !right) { + return false + } + const a = left.endsWith('/') ? left.slice(0, -1) : left + const b = right.endsWith('/') ? right.slice(0, -1) : right + return a === b +} + +function validateCatalog (catalog, { volumeMappings, tmpfs } = {}) { + if (catalog == null) { + return + } + if (catalog.permissions != null && catalog.permissions !== 'ro' && catalog.permissions !== 'rw') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_CATALOG_PERMISSIONS) + } + const items = Array.isArray(catalog.items) ? catalog.items : [] + if (items.length === 0) { + return + } + if (!catalog.bindPath) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_REQUIRED) + } + if (!isAbsoluteContainerPath(catalog.bindPath) && catalog.bindPath !== '/') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE) + } + + const seen = new Set() + for (const item of items) { + const name = item && item.name + if (!name || !SERVICE_NAME_RE.test(name)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_ITEM_NAME, name || '')) + } + if (seen.has(name)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_DUPLICATE_ITEM, name)) + } + seen.add(name) + } + + const mountPaths = collectMountPaths(volumeMappings, tmpfs) + const catalogPaths = [catalog.bindPath, ...items.map((item) => joinCatalogItemPath(catalog.bindPath, item.name))] + for (const catalogPath of catalogPaths) { + for (const mountPath of mountPaths) { + if (pathsCollide(catalogPath, mountPath)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_PATH_COLLISION, catalogPath)) + } + } + } +} + +function pickValue (spec, existing, key) { + if (spec && spec[key] !== undefined) { + return spec[key] + } + return existing ? existing[key] : undefined +} + +function assertPositiveNumber (name, value) { + if (value == null || value === '') { + return + } + if (typeof value !== 'number' || Number.isNaN(value) || value <= 0) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_POSITIVE_NUMBER, name)) + } +} + +function isIpcSysctl (name) { + return name.startsWith('kernel.shm') || + name.startsWith('kernel.msg') || + name.startsWith('kernel.sem') || + name.startsWith('fs.mqueue.') +} + +function coerceSysctlValue (value) { + if (typeof value === 'string') { + return value + } + if (typeof value === 'number' && Number.isFinite(value)) { + return String(value) + } + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_SHAPE) +} + +function normalizeSysctls (sysctls) { + if (sysctls == null || sysctls === '') { + return sysctls + } + if (typeof sysctls !== 'object' || Array.isArray(sysctls)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_SHAPE) + } + const normalized = {} + for (const [key, value] of Object.entries(sysctls)) { + normalized[key] = coerceSysctlValue(value) + } + return normalized +} + +function validateSysctls (sysctls, hostNetworkMode, ipcMode) { + if (sysctls == null || sysctls === '') { + return + } + if (typeof sysctls !== 'object' || Array.isArray(sysctls)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_SHAPE) + } + for (const key of Object.keys(sysctls)) { + if (!SYSCTL_ALLOWLIST.has(key)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_SYSCTL_UNKNOWN, key)) + } + if (hostNetworkMode === true && key.startsWith('net.')) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_HOST_NETWORK) + } + if (ipcMode === 'host' && isIpcSysctl(key)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_HOST_IPC) + } + coerceSysctlValue(sysctls[key]) + } +} + +function validateUlimits (ulimits) { + if (ulimits == null || ulimits === '') { + return + } + if (typeof ulimits !== 'object' || Array.isArray(ulimits)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_ULIMIT_SHAPE_ROOT) + } + for (const [name, value] of Object.entries(ulimits)) { + if (!ULIMIT_ALLOWLIST.has(name)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_UNKNOWN, name)) + } + if (!value || typeof value !== 'object' || Array.isArray(value) || + !Object.prototype.hasOwnProperty.call(value, 'soft') || + !Object.prototype.hasOwnProperty.call(value, 'hard')) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_SHAPE, name)) + } + const soft = value.soft + const hard = value.hard + if (typeof soft !== 'number' || typeof hard !== 'number') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_SHAPE, name)) + } + if (soft === -1 && hard !== -1) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_RANGE, name)) + } + if (soft !== -1 && hard !== -1 && soft > hard) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_RANGE, name)) + } + } +} + +function normalizeDevicePermissions (permissions) { + if (permissions == null || permissions === '') { + return 'rwm' + } + if (typeof permissions !== 'string' || !/^[rwm]+$/.test(permissions)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICE_PERMISSIONS) + } + return permissions +} + +function validateDevices (devices) { + if (devices == null || devices === '') { + return + } + if (!Array.isArray(devices)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICES_SHAPE) + } + for (const device of devices) { + if (!device || typeof device !== 'object') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICES_SHAPE) + } + if (typeof device.hostPath !== 'string' || !device.hostPath.startsWith('/dev/')) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICE_HOST_PATH) + } + if (typeof device.containerPath !== 'string' || device.containerPath.length === 0) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICE_CONTAINER_PATH) + } + device.permissions = normalizeDevicePermissions(device.permissions) + } +} + +function validateTmpfs (tmpfs) { + if (tmpfs == null || tmpfs === '') { + return + } + if (!Array.isArray(tmpfs)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_SHAPE) + } + for (const entry of tmpfs) { + if (!entry || typeof entry !== 'object') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_SHAPE) + } + if (!isAbsoluteContainerPath(entry.containerPath) && entry.containerPath !== '/') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_PATH) + } + if (entry.size != null && (typeof entry.size !== 'number' || entry.size <= 0)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_SIZE) + } + } +} + +function validateContainerFields (spec, existing = {}) { + const data = spec || {} + assertArgv('entrypoint', data.entrypoint) + const argv = resolveProcessArgv(data) + assertArgv('commands', argv) + + const runAsUser = pickValue(data, existing, 'runAsUser') + const runAsGroup = pickValue(data, existing, 'runAsGroup') + if (typeof runAsUser === 'string' && runAsUser.includes(':') && runAsGroup) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_RUN_AS_USER_GROUP_CONFLICT) + } + + const workingDir = data.workingDir + if (workingDir != null && workingDir !== '' && !isAbsoluteContainerPath(workingDir) && workingDir !== '/') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_WORKING_DIR_ABSOLUTE) + } + + assertPositiveNumber('cpus', data.cpus) + assertPositiveNumber('memoryReservation', data.memoryReservation) + assertPositiveNumber('shmSize', data.shmSize) + + const memoryLimit = pickValue(data, existing, 'memoryLimit') + if (data.memorySwap != null && data.memorySwap !== '') { + if (typeof data.memorySwap !== 'number' || Number.isNaN(data.memorySwap)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_MEMORY_SWAP_VALUE) + } + if (data.memorySwap !== -1) { + if (data.memorySwap <= 0) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_MEMORY_SWAP_VALUE) + } + if (memoryLimit == null || memoryLimit === '') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_MEMORY_SWAP_REQUIRES_LIMIT) + } + } + } + + const hostNetworkMode = pickValue(data, existing, 'hostNetworkMode') + const ipcMode = pickValue(data, existing, 'ipcMode') + if (data.sysctls !== undefined) { + data.sysctls = normalizeSysctls(data.sysctls) + } + validateSysctls(data.sysctls, hostNetworkMode, ipcMode) + validateUlimits(data.ulimits) + validateDevices(data.devices) + validateTmpfs(data.tmpfs) +} + +function serializeContainerColumns (spec) { + const data = spec || {} + const columns = {} + if (data.runAsGroup !== undefined) { + columns.runAsGroup = data.runAsGroup + } + if (data.readOnlyRootFilesystem !== undefined) { + columns.readOnlyRootFilesystem = data.readOnlyRootFilesystem + } + if (data.workingDir !== undefined) { + columns.workingDir = data.workingDir + } + if (data.cpus !== undefined) { + columns.cpus = data.cpus + } + if (data.memoryReservation !== undefined) { + columns.memoryReservation = data.memoryReservation + } + if (data.memorySwap !== undefined) { + columns.memorySwap = data.memorySwap + } + if (data.shmSize !== undefined) { + columns.shmSize = data.shmSize + } + if (data.sysctls !== undefined) { + const sysctls = normalizeSysctls(data.sysctls) + columns.sysctls = sysctls && Object.keys(sysctls).length > 0 + ? serializeJsonField(sysctls) + : null + } + return columns +} + +function containerFieldsRequireRebuild (spec, existing) { + if (!spec) { + return false + } + const scalarKeys = [ + 'runAsGroup', 'readOnlyRootFilesystem', 'workingDir', 'cpus', + 'memoryReservation', 'memorySwap', 'shmSize' + ] + for (const key of scalarKeys) { + if (spec[key] !== undefined && spec[key] !== existing[key]) { + return true + } + } + if (spec.entrypoint !== undefined) { + return true + } + if (resolveProcessArgv(spec) !== undefined) { + return true + } + if (spec.sysctls !== undefined || spec.ulimits !== undefined || + spec.devices !== undefined || spec.tmpfs !== undefined) { + return true + } + if (spec.models !== undefined) { + return catalogRequiresRebuild(existing.models, normalizeCatalog(spec.models)) + } + return false +} + +function sortRowsById (rows) { + return Array.isArray(rows) ? rows.slice().sort((left, right) => (left.id || 0) - (right.id || 0)) : [] +} + +function tokensFromRows (rows, field) { + return sortRowsById(rows).map((row) => row[field]).filter((token) => typeof token === 'string') +} + +function devicesFromRows (rows) { + return sortRowsById(rows).map((row) => ({ + hostPath: row.hostPath, + containerPath: row.containerPath, + permissions: row.permissions || 'rwm' + })) +} + +function tmpfsFromRows (rows) { + return sortRowsById(rows).map((row) => { + const entry = { containerPath: row.containerPath } + if (row.size != null) { + entry.size = row.size + } + if (row.mode) { + entry.mode = row.mode + } + return entry + }) +} + +function ulimitsFromRows (rows) { + const ulimits = {} + for (const row of sortRowsById(rows)) { + ulimits[row.name] = { soft: row.soft, hard: row.hard } + } + return ulimits +} + +function catalogFromRows (catalogRow, itemRows) { + const items = sortRowsById(itemRows).map((row) => ({ name: row.name })) + if (!catalogRow && items.length === 0) { + return null + } + return { + bindPath: catalogRow ? catalogRow.bindPath : '', + permissions: (catalogRow && catalogRow.permissions) || 'ro', + items + } +} + +function hydrateJsonColumns (row) { + if (!row) { + return row + } + const hydrated = { ...row } + hydrated.sysctls = parseJsonField(row.sysctls) + return hydrated +} + +function applyAgentContainerFields (response, microservice, extras = {}) { + const cmdFromArgs = Array.isArray(extras) ? extras : extras.cmd + const entrypoint = extras.entrypoint || [] + if (entrypoint.length > 0) { + response.entrypoint = entrypoint + } + + const argv = extras.commands || (Array.isArray(cmdFromArgs) ? cmdFromArgs : []) + if (argv.length > 0) { + response.commands = argv + response.cmd = argv + } else if (Array.isArray(cmdFromArgs)) { + response.cmd = cmdFromArgs + } + + if (microservice.runAsGroup) { + response.runAsGroup = microservice.runAsGroup + } + if (microservice.readOnlyRootFilesystem) { + response.readOnlyRootFilesystem = true + } + if (microservice.workingDir) { + response.workingDir = microservice.workingDir + } + if (microservice.cpus != null) { + response.cpus = microservice.cpus + } + if (microservice.memoryReservation != null) { + response.memoryReservation = microservice.memoryReservation + } + if (microservice.memorySwap != null) { + response.memorySwap = microservice.memorySwap + } + if (microservice.shmSize != null) { + response.shmSize = microservice.shmSize + } + + const sysctls = parseJsonField(microservice.sysctls) + if (sysctls && typeof sysctls === 'object' && Object.keys(sysctls).length > 0) { + response.sysctls = sysctls + } + const ulimits = extras.ulimits + if (ulimits && typeof ulimits === 'object' && Object.keys(ulimits).length > 0) { + response.ulimits = ulimits + } + if (Array.isArray(extras.devices) && extras.devices.length > 0) { + response.devices = extras.devices + } + if (Array.isArray(extras.tmpfs) && extras.tmpfs.length > 0) { + response.tmpfs = extras.tmpfs + } + if (!isCatalogEmpty(extras.models)) { + response.models = extras.models + } + + return response +} + +module.exports = { + parseJsonField, + serializeJsonField, + parseArgv, + serializeArgv, + resolveProcessArgv, + isCatalogEmpty, + parseCatalog, + normalizeCatalog, + serializeCatalog, + catalogItemNames, + catalogRequiresRebuild, + validateCatalog, + validateContainerFields, + serializeContainerColumns, + containerFieldsRequireRebuild, + hydrateJsonColumns, + applyAgentContainerFields, + normalizeSysctls, + tokensFromRows, + devicesFromRows, + tmpfsFromRows, + ulimitsFromRows, + catalogFromRows +} diff --git a/src/routes/microservices.js b/src/routes/microservices.js index c9aa3172..9782126a 100644 --- a/src/routes/microservices.js +++ b/src/routes/microservices.js @@ -261,6 +261,42 @@ module.exports = [ }) } }, + { + method: 'patch', + path: '/api/v3/microservices/:uuid/models', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + + const successCode = constants.HTTP_CODE_NO_CONTENT + const errorCodes = [ + { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] + }, + { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] + }, + { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] + } + ] + + await rbacMiddleware.protect()(req, res, async () => { + const updateMicroserviceCatalogEndPoint = ResponseDecorator.handleErrors(MicroservicesController.updateMicroserviceCatalogEndPoint, + successCode, errorCodes) + const responseObject = await updateMicroserviceCatalogEndPoint(req) + const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' + res + .status(responseObject.code) + .send(responseObject.body) + + logger.apiRes({ req, user, res, responseObject }) + }) + } + }, { method: 'patch', path: '/api/v3/microservices/:uuid/rebuild', diff --git a/src/schemas/microservice-container-spec.js b/src/schemas/microservice-container-spec.js index 6cf86b63..e427f054 100644 --- a/src/schemas/microservice-container-spec.js +++ b/src/schemas/microservice-container-spec.js @@ -8,9 +8,73 @@ const microserviceContainerSpecProperties = { pidMode: { type: 'string' }, ipcMode: { type: 'string' }, runAsUser: { type: 'string' }, + runAsGroup: { type: 'string' }, + readOnlyRootFilesystem: { type: 'boolean' }, platform: { type: 'string' }, cpuSetCpus: { type: 'string' }, memoryLimit: { type: 'integer' }, + memoryReservation: { type: 'integer', minimum: 1 }, + memorySwap: { type: 'integer' }, + shmSize: { type: 'integer', minimum: 1 }, + cpus: { type: 'number' }, + workingDir: { type: 'string' }, + entrypoint: { + type: 'array', + items: { type: 'string' } + }, + commands: { + type: 'array', + items: { type: 'string' } + }, + sysctls: { + type: 'object', + additionalProperties: false, + properties: { + 'kernel.shm_rmid_forced': { type: 'string' }, + 'net.ipv4.ip_local_port_range': { type: 'string' }, + 'net.ipv4.tcp_syncookies': { type: 'string' }, + 'net.ipv4.ping_group_range': { type: 'string' }, + 'net.ipv4.ip_unprivileged_port_start': { type: 'string' }, + 'net.ipv4.ip_local_reserved_ports': { type: 'string' }, + 'net.ipv4.tcp_keepalive_time': { type: 'string' }, + 'net.ipv4.tcp_fin_timeout': { type: 'string' }, + 'net.ipv4.tcp_keepalive_intvl': { type: 'string' }, + 'net.ipv4.tcp_keepalive_probes': { type: 'string' }, + 'net.ipv4.tcp_rmem': { type: 'string' }, + 'net.ipv4.tcp_wmem': { type: 'string' }, + 'net.ipv4.tcp_slow_start_after_idle': { type: 'string' }, + 'net.ipv4.tcp_notsent_lowat': { type: 'string' } + } + }, + ulimits: { + type: 'object', + additionalProperties: false, + properties: { + core: { $ref: '/containerUlimit' }, + cpu: { $ref: '/containerUlimit' }, + data: { $ref: '/containerUlimit' }, + fsize: { $ref: '/containerUlimit' }, + locks: { $ref: '/containerUlimit' }, + memlock: { $ref: '/containerUlimit' }, + msgqueue: { $ref: '/containerUlimit' }, + nice: { $ref: '/containerUlimit' }, + nofile: { $ref: '/containerUlimit' }, + nproc: { $ref: '/containerUlimit' }, + rss: { $ref: '/containerUlimit' }, + rtprio: { $ref: '/containerUlimit' }, + rttime: { $ref: '/containerUlimit' }, + sigpending: { $ref: '/containerUlimit' }, + stack: { $ref: '/containerUlimit' } + } + }, + devices: { + type: 'array', + items: { $ref: '/containerDevice' } + }, + tmpfs: { + type: 'array', + items: { $ref: '/containerTmpfs' } + }, ports: { type: 'array', items: { $ref: '/ports' } diff --git a/src/schemas/microservice.js b/src/schemas/microservice.js index 76dc34bb..423c6e09 100644 --- a/src/schemas/microservice.js +++ b/src/schemas/microservice.js @@ -1,4 +1,4 @@ -const { nameRegex } = require('./utils/utils') +const { nameRegex, serviceNameRegex } = require('./utils/utils') const microserviceCreate = { id: '/microserviceCreate', @@ -66,10 +66,75 @@ const microserviceCreate = { items: { type: 'string' } }, runAsUser: { type: 'string' }, + runAsGroup: { type: 'string' }, + readOnlyRootFilesystem: { type: 'boolean' }, platform: { type: 'string' }, runtime: { type: 'string' }, cpuSetCpus: { type: 'string' }, memoryLimit: { type: 'integer' }, + memoryReservation: { type: 'integer', minimum: 1 }, + memorySwap: { type: 'integer' }, + shmSize: { type: 'integer', minimum: 1 }, + cpus: { type: 'number' }, + workingDir: { type: 'string' }, + entrypoint: { + type: 'array', + items: { type: 'string' } + }, + commands: { + type: 'array', + items: { type: 'string' } + }, + sysctls: { + type: 'object', + additionalProperties: false, + properties: { + 'kernel.shm_rmid_forced': { type: 'string' }, + 'net.ipv4.ip_local_port_range': { type: 'string' }, + 'net.ipv4.tcp_syncookies': { type: 'string' }, + 'net.ipv4.ping_group_range': { type: 'string' }, + 'net.ipv4.ip_unprivileged_port_start': { type: 'string' }, + 'net.ipv4.ip_local_reserved_ports': { type: 'string' }, + 'net.ipv4.tcp_keepalive_time': { type: 'string' }, + 'net.ipv4.tcp_fin_timeout': { type: 'string' }, + 'net.ipv4.tcp_keepalive_intvl': { type: 'string' }, + 'net.ipv4.tcp_keepalive_probes': { type: 'string' }, + 'net.ipv4.tcp_rmem': { type: 'string' }, + 'net.ipv4.tcp_wmem': { type: 'string' }, + 'net.ipv4.tcp_slow_start_after_idle': { type: 'string' }, + 'net.ipv4.tcp_notsent_lowat': { type: 'string' } + } + }, + ulimits: { + type: 'object', + additionalProperties: false, + properties: { + core: { $ref: '/containerUlimit' }, + cpu: { $ref: '/containerUlimit' }, + data: { $ref: '/containerUlimit' }, + fsize: { $ref: '/containerUlimit' }, + locks: { $ref: '/containerUlimit' }, + memlock: { $ref: '/containerUlimit' }, + msgqueue: { $ref: '/containerUlimit' }, + nice: { $ref: '/containerUlimit' }, + nofile: { $ref: '/containerUlimit' }, + nproc: { $ref: '/containerUlimit' }, + rss: { $ref: '/containerUlimit' }, + rtprio: { $ref: '/containerUlimit' }, + rttime: { $ref: '/containerUlimit' }, + sigpending: { $ref: '/containerUlimit' }, + stack: { $ref: '/containerUlimit' } + } + }, + devices: { + type: 'array', + items: { $ref: '/containerDevice' } + }, + tmpfs: { + type: 'array', + items: { $ref: '/containerTmpfs' } + }, + models: { $ref: '/microserviceCatalog' }, natsConfig: { $ref: '/microserviceNatsConfig' }, healthCheck: { type: 'object', @@ -155,10 +220,75 @@ const microserviceUpdate = { items: { type: 'string' } }, runAsUser: { type: 'string' }, + runAsGroup: { type: 'string' }, + readOnlyRootFilesystem: { type: 'boolean' }, platform: { type: 'string' }, runtime: { type: 'string' }, cpuSetCpus: { type: 'string' }, memoryLimit: { type: 'integer' }, + memoryReservation: { type: 'integer', minimum: 1 }, + memorySwap: { type: 'integer' }, + shmSize: { type: 'integer', minimum: 1 }, + cpus: { type: 'number' }, + workingDir: { type: 'string' }, + entrypoint: { + type: 'array', + items: { type: 'string' } + }, + commands: { + type: 'array', + items: { type: 'string' } + }, + sysctls: { + type: 'object', + additionalProperties: false, + properties: { + 'kernel.shm_rmid_forced': { type: 'string' }, + 'net.ipv4.ip_local_port_range': { type: 'string' }, + 'net.ipv4.tcp_syncookies': { type: 'string' }, + 'net.ipv4.ping_group_range': { type: 'string' }, + 'net.ipv4.ip_unprivileged_port_start': { type: 'string' }, + 'net.ipv4.ip_local_reserved_ports': { type: 'string' }, + 'net.ipv4.tcp_keepalive_time': { type: 'string' }, + 'net.ipv4.tcp_fin_timeout': { type: 'string' }, + 'net.ipv4.tcp_keepalive_intvl': { type: 'string' }, + 'net.ipv4.tcp_keepalive_probes': { type: 'string' }, + 'net.ipv4.tcp_rmem': { type: 'string' }, + 'net.ipv4.tcp_wmem': { type: 'string' }, + 'net.ipv4.tcp_slow_start_after_idle': { type: 'string' }, + 'net.ipv4.tcp_notsent_lowat': { type: 'string' } + } + }, + ulimits: { + type: 'object', + additionalProperties: false, + properties: { + core: { $ref: '/containerUlimit' }, + cpu: { $ref: '/containerUlimit' }, + data: { $ref: '/containerUlimit' }, + fsize: { $ref: '/containerUlimit' }, + locks: { $ref: '/containerUlimit' }, + memlock: { $ref: '/containerUlimit' }, + msgqueue: { $ref: '/containerUlimit' }, + nice: { $ref: '/containerUlimit' }, + nofile: { $ref: '/containerUlimit' }, + nproc: { $ref: '/containerUlimit' }, + rss: { $ref: '/containerUlimit' }, + rtprio: { $ref: '/containerUlimit' }, + rttime: { $ref: '/containerUlimit' }, + sigpending: { $ref: '/containerUlimit' }, + stack: { $ref: '/containerUlimit' } + } + }, + devices: { + type: 'array', + items: { $ref: '/containerDevice' } + }, + tmpfs: { + type: 'array', + items: { $ref: '/containerTmpfs' } + }, + models: { $ref: '/microserviceCatalog' }, natsConfig: { $ref: '/microserviceNatsConfig' }, healthCheck: { type: 'object', @@ -294,7 +424,85 @@ const microserviceNatsConfig = { additionalProperties: false } +const microserviceCatalogItem = { + id: '/microserviceCatalogItem', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + } + }, + required: ['name'], + additionalProperties: false +} + +const microserviceCatalog = { + id: '/microserviceCatalog', + type: 'object', + properties: { + bindPath: { type: 'string' }, + permissions: { enum: ['ro', 'rw'] }, + items: { + type: 'array', + items: { $ref: '/microserviceCatalogItem' } + } + }, + additionalProperties: false +} + +const microserviceCatalogPatch = { + id: '/microserviceCatalogPatch', + type: 'object', + properties: { + bindPath: { type: 'string' }, + permissions: { enum: ['ro', 'rw'] }, + items: { + type: 'array', + items: { $ref: '/microserviceCatalogItem' } + } + }, + additionalProperties: false +} + +const containerDevice = { + id: '/containerDevice', + type: 'object', + properties: { + hostPath: { type: 'string', pattern: '^/dev/' }, + containerPath: { type: 'string', minLength: 1 }, + permissions: { type: 'string', pattern: '^[rwm]+$' } + }, + required: ['hostPath', 'containerPath'], + additionalProperties: false +} + +const containerTmpfs = { + id: '/containerTmpfs', + type: 'object', + properties: { + containerPath: { type: 'string', minLength: 1 }, + size: { type: 'integer', minimum: 1 }, + mode: { type: 'string' } + }, + required: ['containerPath'], + additionalProperties: false +} + +const containerUlimit = { + id: '/containerUlimit', + type: 'object', + properties: { + soft: { type: 'integer' }, + hard: { type: 'integer' } + }, + required: ['soft', 'hard'], + additionalProperties: false +} + module.exports = { - mainSchemas: [microserviceCreate, microserviceUpdate, env, ports, extraHosts, portsCreate, microserviceDelete, volumeMappings, microserviceHealthCheck], - innerSchemas: [volumeMappings, ports, env, extraHosts, microserviceCreate, microserviceHealthCheck, microserviceNatsConfig] + mainSchemas: [microserviceCreate, microserviceUpdate, env, ports, extraHosts, portsCreate, microserviceDelete, volumeMappings, microserviceHealthCheck, microserviceCatalogPatch], + innerSchemas: [volumeMappings, ports, env, extraHosts, microserviceCreate, microserviceHealthCheck, microserviceNatsConfig, microserviceCatalog, microserviceCatalogItem, containerDevice, containerTmpfs, containerUlimit] } diff --git a/src/services/event-service.js b/src/services/event-service.js index cdf26950..a862ca31 100644 --- a/src/services/event-service.js +++ b/src/services/event-service.js @@ -29,6 +29,9 @@ function extractResourceType (path) { { pattern: /^\/api\/v3\/nats/, type: 'nats' }, { pattern: /^\/api\/v3\/registries/, type: 'registry' }, { pattern: /^\/api\/v3\/volumeMounts/, type: 'volumeMount' }, + { pattern: /^\/api\/v3\/models/, type: 'model' }, + { pattern: /^\/api\/v3\/runtimeClasses/, type: 'runtimeClass' }, + { pattern: /^\/api\/v3\/microserviceTemplates/, type: 'microserviceTemplate' }, { pattern: /^\/api\/v3\/configMaps/, type: 'configMap' }, { pattern: /^\/api\/v3\/applicationTemplates/, type: 'applicationTemplate' }, { pattern: /^\/api\/v3\/catalog/, type: 'catalog' }, diff --git a/src/services/microservice-workload-spec.js b/src/services/microservice-workload-spec.js index 599ab119..636e5e19 100644 --- a/src/services/microservice-workload-spec.js +++ b/src/services/microservice-workload-spec.js @@ -2,10 +2,15 @@ const AppHelper = require('../helpers/app-helper') const Errors = require('../helpers/errors') const ErrorMessages = require('../helpers/error-messages') const constants = require('../helpers/constants') +const CatalogContainer = require('../helpers/microservice-container-catalog') const FogManager = require('../data/managers/iofog-manager') const ApplicationManager = require('../data/managers/application-manager') const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceArgManager = require('../data/managers/microservice-arg-manager') +const MicroserviceEntrypointManager = require('../data/managers/microservice-entrypoint-manager') +const MicroserviceDeviceManager = require('../data/managers/microservice-device-manager') +const MicroserviceTmpfsManager = require('../data/managers/microservice-tmpfs-manager') +const MicroserviceUlimitManager = require('../data/managers/microservice-ulimit-manager') const MicroserviceCdiDevManager = require('../data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../data/managers/microservice-cap-drop-manager') @@ -64,7 +69,8 @@ function buildScalarColumns (spec, { defaultLogSize = constants.MICROSERVICE_DEF runAsUser: spec.runAsUser, platform: spec.platform, cpuSetCpus: spec.cpuSetCpus, - memoryLimit: spec.memoryLimit + memoryLimit: spec.memoryLimit, + ...CatalogContainer.serializeContainerColumns(spec) }) } @@ -255,6 +261,61 @@ async function _updateArg (arg, microserviceUuid, transaction) { } } +async function _replaceEntrypoint (entrypoint, microserviceUuid, transaction) { + await MicroserviceEntrypointManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(entrypoint)) { + return + } + for (const token of entrypoint) { + await MicroserviceEntrypointManager.create({ entrypoint: token, microserviceUuid }, transaction) + } +} + +async function _replaceDevices (devices, microserviceUuid, transaction) { + await MicroserviceDeviceManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(devices)) { + return + } + for (const device of devices) { + await MicroserviceDeviceManager.create({ + hostPath: device.hostPath, + containerPath: device.containerPath, + permissions: device.permissions || 'rwm', + microserviceUuid + }, transaction) + } +} + +async function _replaceTmpfs (tmpfs, microserviceUuid, transaction) { + await MicroserviceTmpfsManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(tmpfs)) { + return + } + for (const entry of tmpfs) { + await MicroserviceTmpfsManager.create({ + containerPath: entry.containerPath, + size: entry.size, + mode: entry.mode, + microserviceUuid + }, transaction) + } +} + +async function _replaceUlimits (ulimits, microserviceUuid, transaction) { + await MicroserviceUlimitManager.delete({ microserviceUuid }, transaction) + if (!ulimits || typeof ulimits !== 'object') { + return + } + for (const [name, value] of Object.entries(ulimits)) { + await MicroserviceUlimitManager.create({ + name, + soft: value.soft, + hard: value.hard, + microserviceUuid + }, transaction) + } +} + async function _createCdiDevice (microserviceUuid, cdiDevice, transaction) { await MicroserviceCdiDevManager.create({ cdiDevices: cdiDevice, microserviceUuid }, transaction) } @@ -306,6 +367,7 @@ async function _updateHealthCheck (microserviceUuid, healthCheck, transaction) { } async function createWorkloadRelations (microservice, spec, { validatedExtraHosts, transaction }) { + CatalogContainer.validateContainerFields(spec, {}) const microserviceUuid = microservice.uuid if (validatedExtraHosts && validatedExtraHosts.length) { @@ -320,12 +382,26 @@ async function createWorkloadRelations (microservice, spec, { validatedExtraHost } } - if (spec.cmd) { - for (const arg of spec.cmd) { + const argv = CatalogContainer.resolveProcessArgv(spec) + if (argv) { + for (const arg of argv) { await _createArg(microserviceUuid, arg, transaction) } } + if (spec.entrypoint !== undefined) { + await _replaceEntrypoint(spec.entrypoint, microserviceUuid, transaction) + } + if (spec.devices !== undefined) { + await _replaceDevices(spec.devices, microserviceUuid, transaction) + } + if (spec.tmpfs !== undefined) { + await _replaceTmpfs(spec.tmpfs, microserviceUuid, transaction) + } + if (spec.ulimits !== undefined) { + await _replaceUlimits(spec.ulimits, microserviceUuid, transaction) + } + if (spec.cdiDevices) { for (const cdiDevice of spec.cdiDevices) { await _createCdiDevice(microserviceUuid, cdiDevice, transaction) @@ -350,6 +426,8 @@ async function createWorkloadRelations (microservice, spec, { validatedExtraHost } async function updateWorkloadRelations (microserviceUuid, spec, { validatedExtraHosts, transaction }) { + CatalogContainer.validateContainerFields(spec, {}) + if (validatedExtraHosts) { await _updateExtraHosts(validatedExtraHosts, microserviceUuid, transaction) } @@ -358,8 +436,22 @@ async function updateWorkloadRelations (microserviceUuid, spec, { validatedExtra await _updateEnv(spec.env, microserviceUuid, transaction) } - if (spec.cmd) { - await _updateArg(spec.cmd, microserviceUuid, transaction) + const argv = CatalogContainer.resolveProcessArgv(spec) + if (argv !== undefined) { + await _updateArg(argv, microserviceUuid, transaction) + } + + if (spec.entrypoint !== undefined) { + await _replaceEntrypoint(spec.entrypoint, microserviceUuid, transaction) + } + if (spec.devices !== undefined) { + await _replaceDevices(spec.devices, microserviceUuid, transaction) + } + if (spec.tmpfs !== undefined) { + await _replaceTmpfs(spec.tmpfs, microserviceUuid, transaction) + } + if (spec.ulimits !== undefined) { + await _replaceUlimits(spec.ulimits, microserviceUuid, transaction) } if (spec.cdiDevices) { @@ -400,10 +492,23 @@ function shouldRebuildForWorkloadChange (existing, spec, { config, annotations, spec.ports || spec.extraHosts || spec.cmd || + spec.commands || + spec.entrypoint || spec.cdiDevices || spec.capAdd || spec.capDrop || - spec.healthCheck + spec.healthCheck || + spec.runAsGroup || + spec.readOnlyRootFilesystem || + spec.workingDir || + spec.cpus || + spec.memoryReservation || + spec.memorySwap || + spec.shmSize || + spec.sysctls || + spec.ulimits || + spec.devices || + spec.tmpfs ) } diff --git a/src/services/microservices-service.js b/src/services/microservices-service.js index 00c64cd9..c9cfa819 100644 --- a/src/services/microservices-service.js +++ b/src/services/microservices-service.js @@ -3,6 +3,12 @@ const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const MicroserviceArgManager = require('../data/managers/microservice-arg-manager') +const MicroserviceEntrypointManager = require('../data/managers/microservice-entrypoint-manager') +const MicroserviceDeviceManager = require('../data/managers/microservice-device-manager') +const MicroserviceTmpfsManager = require('../data/managers/microservice-tmpfs-manager') +const MicroserviceUlimitManager = require('../data/managers/microservice-ulimit-manager') +const MicroserviceModelManager = require('../data/managers/microservice-model-manager') +const MicroserviceModelItemManager = require('../data/managers/microservice-model-item-manager') const MicroserviceCdiDevManager = require('../data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../data/managers/microservice-cap-drop-manager') @@ -33,6 +39,11 @@ const ServiceServices = require('./services-service') const ConfigMapManager = require('../data/managers/config-map-manager') const SecretManager = require('../data/managers/secret-manager') const VolumeMountService = require('./volume-mount-service') +const RuntimeClassService = require('./runtime-class-service') +const ModelService = require('./model-service') +const RegistryService = require('./registry-service') +const FleetModelManager = require('../data/managers/fleet-model-manager') +const CatalogContainer = require('../helpers/microservice-container-catalog') const RbacServiceAccountManager = require('../data/managers/rbac-service-account-manager') const RbacRoleManager = require('../data/managers/rbac-role-manager') const NatsAuthService = require('./nats-auth-service') @@ -51,6 +62,23 @@ const logger = require('../logger') const SERVICE_ACCOUNT_VOLUME_TYPE = 'serviceAccount' const SERVICE_ACCOUNT_VOLUME_CONTAINER_DESTINATION = '/var/run/secrets/edgelet.iofog.org/serviceaccount' const SERVICE_ACCOUNT_VOLUME_ACCESS_MODE = 'ro' + +async function _resolveServiceAccountForUserResponse (microservice, transaction) { + let serviceAccount = microservice.serviceAccount + if (!serviceAccount && microservice.uuid) { + serviceAccount = await RbacServiceAccountManager.findOneByMicroserviceUuid(microservice.uuid, transaction) + } + if (!serviceAccount) { + return null + } + const plain = serviceAccount.get ? serviceAccount.get({ plain: true }) : serviceAccount + if (!plain.roleRef) { + return null + } + return { + roleRef: plain.roleRef + } +} const NATS_CREDS_PATH_ENV = 'NATS_CREDS_PATH' const NATS_SERVER_URL_ENV = 'NATS_SERVER_URL' @@ -425,7 +453,7 @@ function _parseFogAvailableRuntimes (fog) { } } -function _validateMicroserviceRuntime (runtime, fog) { +async function _validateMicroserviceRuntime (runtime, fog, transaction) { if (runtime == null || runtime === '') { return } @@ -436,6 +464,181 @@ function _validateMicroserviceRuntime (runtime, fog) { `Runtime '${runtime}' is not available on agent '${agentLabel}'` ) } + await RuntimeClassService.ensureRuntimeClassLinkedToFog(fog.uuid, runtime, transaction) +} + +async function _assertOciRegistryForImage (registryId, transaction) { + if (registryId == null) { + return + } + await RegistryService.assertOciRegistryForImage(registryId, transaction) +} + +async function _assertCatalogModelsExist (catalog, transaction) { + const names = CatalogContainer.catalogItemNames(catalog) + for (const name of names) { + const model = await FleetModelManager.findOne({ name }, transaction) + if (!model) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_MODEL_NOT_FOUND, name)) + } + } +} + +async function _autoAttachCatalogModels (fogUuid, catalog, transaction) { + if (!fogUuid) { + return + } + const names = CatalogContainer.catalogItemNames(catalog) + if (names.length === 0) { + return + } + await ModelService.ensureModelsLinkedToFog(fogUuid, names, transaction) +} + +async function _loadCatalog (microserviceUuid, transaction) { + const catalogRow = await MicroserviceModelManager.findOne({ microserviceUuid }, transaction) + const itemRows = await MicroserviceModelItemManager.findAll({ microserviceUuid }, transaction) + return CatalogContainer.catalogFromRows(catalogRow, itemRows) +} + +async function _loadTmpfs (microserviceUuid, transaction) { + const rows = await MicroserviceTmpfsManager.findAll({ microserviceUuid }, transaction) + return CatalogContainer.tmpfsFromRows(rows) +} + +function _prepareCatalogAndContainerFields (microserviceData, existing = {}, volumeMappings, tmpfs) { + CatalogContainer.validateContainerFields(microserviceData, existing) + if (microserviceData.models !== undefined) { + const catalog = CatalogContainer.normalizeCatalog(microserviceData.models) + CatalogContainer.validateCatalog(catalog, { + volumeMappings: volumeMappings !== undefined ? volumeMappings : existing.volumeMappings, + tmpfs: tmpfs !== undefined ? tmpfs : (microserviceData.tmpfs !== undefined ? microserviceData.tmpfs : existing.tmpfs) + }) + } +} + +function _assignContainerColumns (target, microserviceData) { + Object.assign(target, CatalogContainer.serializeContainerColumns(microserviceData)) +} + +async function _replaceEntrypoint (entrypoint, microserviceUuid, transaction) { + await MicroserviceEntrypointManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(entrypoint)) { + return + } + for (const token of entrypoint) { + await MicroserviceEntrypointManager.create({ + entrypoint: token, + microserviceUuid + }, transaction) + } +} + +async function _replaceDevices (devices, microserviceUuid, transaction) { + await MicroserviceDeviceManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(devices)) { + return + } + for (const device of devices) { + await MicroserviceDeviceManager.create({ + hostPath: device.hostPath, + containerPath: device.containerPath, + permissions: device.permissions || 'rwm', + microserviceUuid + }, transaction) + } +} + +async function _replaceTmpfs (tmpfs, microserviceUuid, transaction) { + await MicroserviceTmpfsManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(tmpfs)) { + return + } + for (const entry of tmpfs) { + await MicroserviceTmpfsManager.create({ + containerPath: entry.containerPath, + size: entry.size, + mode: entry.mode, + microserviceUuid + }, transaction) + } +} + +async function _replaceUlimits (ulimits, microserviceUuid, transaction) { + await MicroserviceUlimitManager.delete({ microserviceUuid }, transaction) + if (!ulimits || typeof ulimits !== 'object') { + return + } + for (const [name, value] of Object.entries(ulimits)) { + await MicroserviceUlimitManager.create({ + name, + soft: value.soft, + hard: value.hard, + microserviceUuid + }, transaction) + } +} + +async function _replaceCatalog (catalogInput, microserviceUuid, transaction) { + await MicroserviceModelItemManager.delete({ microserviceUuid }, transaction) + await MicroserviceModelManager.delete({ microserviceUuid }, transaction) + const catalog = CatalogContainer.normalizeCatalog(catalogInput) + if (CatalogContainer.isCatalogEmpty(catalog)) { + return + } + await MicroserviceModelManager.create({ + bindPath: catalog.bindPath, + permissions: catalog.permissions || 'ro', + microserviceUuid + }, transaction) + for (const item of catalog.items) { + await MicroserviceModelItemManager.create({ + name: item.name, + microserviceUuid + }, transaction) + } +} + +async function _createContainerChildRows (microservice, microserviceData, transaction) { + if (microserviceData.entrypoint !== undefined) { + await _replaceEntrypoint(microserviceData.entrypoint, microservice.uuid, transaction) + } + if (microserviceData.devices !== undefined) { + await _replaceDevices(microserviceData.devices, microservice.uuid, transaction) + } + if (microserviceData.tmpfs !== undefined) { + await _replaceTmpfs(microserviceData.tmpfs, microservice.uuid, transaction) + } + if (microserviceData.ulimits !== undefined) { + await _replaceUlimits(microserviceData.ulimits, microservice.uuid, transaction) + } + if (microserviceData.models !== undefined) { + await _replaceCatalog(microserviceData.models, microservice.uuid, transaction) + } +} + +async function _applyCatalogContainerUpdate (microserviceData, microservice, microserviceDataUpdate, fogUuid, transaction) { + const volumeMappings = microserviceData.volumeMappings !== undefined + ? microserviceData.volumeMappings + : await VolumeMappingManager.findAll({ microserviceUuid: microservice.uuid }, transaction) + const tmpfs = microserviceData.tmpfs !== undefined + ? microserviceData.tmpfs + : await _loadTmpfs(microservice.uuid, transaction) + const existingCatalog = await _loadCatalog(microservice.uuid, transaction) + microservice.models = existingCatalog + _prepareCatalogAndContainerFields(microserviceData, microservice, volumeMappings, tmpfs) + _assignContainerColumns(microserviceDataUpdate, microserviceData) + const argv = CatalogContainer.resolveProcessArgv(microserviceData) + if (argv !== undefined) { + microserviceDataUpdate.cmd = argv + } + await _assertOciRegistryForImage(microserviceDataUpdate.registryId, transaction) + if (microserviceData.models !== undefined) { + const catalog = CatalogContainer.normalizeCatalog(microserviceData.models) + await _assertCatalogModelsExist(catalog, transaction) + await _autoAttachCatalogModels(fogUuid, catalog, transaction) + } + await _createContainerChildRows(microservice, microserviceData, transaction) } function _isServiceAccountVolumeType (type) { @@ -520,7 +723,48 @@ async function _normalizeMicroserviceNatsConfig (microserviceData, transaction, } } +function _identityOverlayFromRequest (request, existing = null) { + const overlay = {} + if (request.name !== undefined) { + overlay.name = request.name + } else if (existing?.name) { + overlay.name = existing.name + } + if (request.application !== undefined) { + overlay.application = request.application + } + if (request.iofogUuid !== undefined) { + overlay.iofogUuid = request.iofogUuid + } else if (existing?.iofogUuid) { + overlay.iofogUuid = existing.iofogUuid + } + if (request.agentName !== undefined) { + overlay.agentName = request.agentName + } + return overlay +} + +async function _applyMicroserviceTemplateOverlay (microserviceData, isCLI, transaction, existing = null) { + if (!microserviceData.template || !microserviceData.template.name) { + return microserviceData + } + + const MicroserviceTemplateService = require('./microservice-template-service') + const identity = _identityOverlayFromRequest(microserviceData, existing) + const fromTemplate = await MicroserviceTemplateService.getMicroserviceDataFromTemplate( + microserviceData.template, + isCLI, + transaction + ) + return { + ...fromTemplate, + ...identity + } +} + async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) { + microserviceData = await _applyMicroserviceTemplateOverlay(microserviceData, isCLI, transaction) + // API Retro compatibility if (!microserviceData.application) { microserviceData.application = microserviceData.flowId @@ -537,7 +781,7 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) // Set fog uuid for further reference microserviceData.iofogUuid = fog.uuid - _validateMicroserviceRuntime(microserviceData.runtime, fog) + await _validateMicroserviceRuntime(microserviceData.runtime, fog, transaction) // validate images if (microserviceData.catalogItemId) { @@ -559,6 +803,8 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_DOES_NOT_HAVE_IMAGES, microserviceData.name)) } + await _assertOciRegistryForImage(microserviceData.registryId || 1, transaction) + // validate extraHosts const extraHosts = await _validateExtraHosts(microserviceData, fog.uuid, transaction) @@ -566,6 +812,11 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) _validateVolumeMappings(microserviceData.volumeMappings) + _prepareCatalogAndContainerFields(microserviceData, {}, microserviceData.volumeMappings, microserviceData.tmpfs) + if (microserviceData.models !== undefined) { + await _assertCatalogModelsExist(CatalogContainer.normalizeCatalog(microserviceData.models), transaction) + } + const microservice = await _createMicroservice({ ...microserviceData, iofogUuid: fog.uuid }, isCLI, transaction) if (!microserviceData.catalogItemId) { @@ -605,8 +856,9 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) await _createEnv(microservice, env, transaction) } } - if (microserviceData.cmd) { - for (const arg of microserviceData.cmd) { + const createArgv = CatalogContainer.resolveProcessArgv(microserviceData) + if (createArgv) { + for (const arg of createArgv) { await _createArg(microservice, arg, transaction) } } @@ -638,7 +890,14 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) await _injectServiceAccountVolume(microservice, transaction) + await _createContainerChildRows(microservice, microserviceData, transaction) + + if (microserviceData.models !== undefined) { + await _autoAttachCatalogModels(fog.uuid, CatalogContainer.normalizeCatalog(microserviceData.models), transaction) + } + if (microserviceData.iofogUuid) { + // Full create always reloads the microservice list on the agent. await _updateChangeTracking(false, microserviceData.iofogUuid, transaction) } @@ -936,6 +1195,14 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD microserviceDataUpdate.registryId = microservice.registryId } + await _applyCatalogContainerUpdate( + microserviceData, + microservice, + microserviceDataUpdate, + microserviceDataUpdate.iofogUuid || microservice.iofogUuid, + transaction + ) + if (microserviceDataUpdate.ports) { await _updateSystemPorts(microserviceDataUpdate.ports, microservice, transaction) } @@ -1023,7 +1290,7 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD const runtimeToValidate = microserviceDataUpdate.runtime !== undefined ? microserviceDataUpdate.runtime : microservice.runtime - _validateMicroserviceRuntime(runtimeToValidate, fog) + await _validateMicroserviceRuntime(runtimeToValidate, fog, transaction) } } @@ -1048,7 +1315,7 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD microserviceDataUpdate.ports || (microserviceDataUpdate.schedule !== undefined && microserviceDataUpdate.schedule !== microservice.schedule) || extraHosts - ) + ) || CatalogContainer.containerFieldsRequireRebuild(microserviceData, microservice) const updatedMicroservice = await MicroserviceManager.updateAndFind(query, microserviceDataUpdate, transaction) if (extraHosts) { @@ -1152,6 +1419,7 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, isCLI, transaction, changeTrackingEnabled = true) { const current = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) + microserviceData = await _applyMicroserviceTemplateOverlay(microserviceData, isCLI, transaction, current) await _normalizeMicroserviceNatsConfig(microserviceData, transaction, current) await Validator.validate(microserviceData, Validator.schemas.microserviceUpdate) let needStatusReset = false @@ -1327,10 +1595,18 @@ async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, i const runtimeToValidate = microserviceDataUpdate.runtime !== undefined ? microserviceDataUpdate.runtime : microservice.runtime - _validateMicroserviceRuntime(runtimeToValidate, fog) + await _validateMicroserviceRuntime(runtimeToValidate, fog, transaction) } } + await _applyCatalogContainerUpdate( + microserviceData, + microservice, + microserviceDataUpdate, + microserviceDataUpdate.iofogUuid || microservice.iofogUuid, + transaction + ) + // Set rebuild flag if needed microserviceDataUpdate.rebuild = microserviceDataUpdate.rebuild || !!( (microserviceDataUpdate.hostNetworkMode !== undefined && microservice.hostNetworkMode !== microserviceDataUpdate.hostNetworkMode) || @@ -1352,7 +1628,7 @@ async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, i microserviceDataUpdate.ports || (microserviceDataUpdate.schedule !== undefined && microserviceDataUpdate.schedule !== microservice.schedule) || extraHosts - ) + ) || CatalogContainer.containerFieldsRequireRebuild(microserviceData, microservice) const updatedMicroservice = await MicroserviceManager.updateAndFind(query, microserviceDataUpdate, transaction) if (extraHosts) { @@ -1501,6 +1777,53 @@ async function updateMicroserviceConfigEndPoint (microserviceUuid, config, isCLI } } +async function updateMicroserviceCatalogEndPoint (microserviceUuid, catalogData, isCLI, transaction) { + await Validator.validate(catalogData, Validator.schemas.microserviceCatalogPatch) + + const query = { uuid: microserviceUuid } + const microservice = await MicroserviceManager.findOneWithCategory(query, transaction) + if (!microservice) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, microserviceUuid)) + } + if (microservice.catalogItem && microservice.catalogItem.category === 'SYSTEM') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (microservice.isController) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (!isCLI) { + await _validateMicroserviceOnGet(microserviceUuid, transaction) + } + + const catalog = CatalogContainer.normalizeCatalog(catalogData) + const volumeMappings = await VolumeMappingManager.findAll({ microserviceUuid }, transaction) + const tmpfs = await _loadTmpfs(microserviceUuid, transaction) + CatalogContainer.validateCatalog(catalog, { + volumeMappings, + tmpfs + }) + await _assertCatalogModelsExist(catalog, transaction) + await _autoAttachCatalogModels(microservice.iofogUuid, catalog, transaction) + + const previousCatalog = await _loadCatalog(microserviceUuid, transaction) + const rebuild = CatalogContainer.catalogRequiresRebuild(previousCatalog, catalog) + await _replaceCatalog(catalog, microserviceUuid, transaction) + await MicroserviceManager.update(query, { + rebuild: rebuild || !!microservice.rebuild + }, transaction) + + if (microservice.iofogUuid) { + const event = rebuild + ? ChangeTrackingService.events.microserviceList + : ChangeTrackingService.events.microserviceModels + await ChangeTrackingService.update(microservice.iofogUuid, event, transaction) + } + + return { + uuid: microserviceUuid + } +} + async function getMicroserviceConfigEndPoint (microserviceUuid, isCLI, transaction) { const query = isCLI ? { @@ -2176,6 +2499,8 @@ async function _createMicroservice (microserviceData, isCLI, transaction) { natsRuleId: microserviceData.natsRuleId } + _assignContainerColumns(newMicroservice, microserviceData) + newMicroservice = AppHelper.deleteUndefinedFields(newMicroservice) if (newMicroservice.registryId) { @@ -2503,6 +2828,7 @@ async function _updateSystemPorts (newPortMappings, microservice, transaction) { async function _updateChangeTracking (configUpdated, fogNodeUuid, transaction) { if (configUpdated) { + // Spec updates other than a catalog-only PATCH reload the full microservice list. await ChangeTrackingService.update(fogNodeUuid, ChangeTrackingService.events.microserviceCommon, transaction) } else { await ChangeTrackingService.update(fogNodeUuid, ChangeTrackingService.events.microserviceList, transaction) @@ -2579,8 +2905,17 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { const images = await CatalogItemImageManager.findAll({ microserviceUuid }, transaction) const volumeMappings = await VolumeMappingManager.findAll({ microserviceUuid }, transaction) const env = await MicroserviceEnvManager.findAllExcludeFields({ microserviceUuid }, transaction) - const cmd = await MicroserviceArgManager.findAllExcludeFields({ microserviceUuid }, transaction) - const arg = cmd.map((it) => it.cmd) + const cmdRows = await MicroserviceArgManager.findAll({ microserviceUuid }, transaction) + const arg = CatalogContainer.tokensFromRows(cmdRows, 'cmd') + const entrypointRows = await MicroserviceEntrypointManager.findAll({ microserviceUuid }, transaction) + const entrypoint = CatalogContainer.tokensFromRows(entrypointRows, 'entrypoint') + const deviceRows = await MicroserviceDeviceManager.findAll({ microserviceUuid }, transaction) + const devices = CatalogContainer.devicesFromRows(deviceRows) + const tmpfsRows = await MicroserviceTmpfsManager.findAll({ microserviceUuid }, transaction) + const tmpfs = CatalogContainer.tmpfsFromRows(tmpfsRows) + const ulimitRows = await MicroserviceUlimitManager.findAll({ microserviceUuid }, transaction) + const ulimits = CatalogContainer.ulimitsFromRows(ulimitRows) + const catalog = await _loadCatalog(microserviceUuid, transaction) const cdiDevices = await MicroserviceCdiDevManager.findAllExcludeFields({ microserviceUuid }, transaction) const cdiDevs = cdiDevices.map((it) => it.cdiDevices) const capAdd = await MicroserviceCapAddManager.findAllExcludeFields({ microserviceUuid }, transaction) @@ -2601,6 +2936,23 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { res.volumeMappings = volumeMappings.map((vm) => vm.dataValues) res.env = env res.cmd = arg + const hydrated = CatalogContainer.hydrateJsonColumns(res) + res.sysctls = hydrated.sysctls + if (entrypoint.length > 0) { + res.entrypoint = entrypoint + } else { + delete res.entrypoint + } + if (arg.length > 0) { + res.commands = arg + } else { + delete res.commands + } + res.ulimits = Object.keys(ulimits).length > 0 ? ulimits : null + res.devices = devices + res.tmpfs = tmpfs + res.models = catalog + res.runAsGroup = res.runAsGroup || '' res.cdiDevices = cdiDevs res.capAdd = capAdds res.capDrop = capDrops @@ -2679,6 +3031,9 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { } delete res.natsRuleId + delete res.serviceAccount + res.serviceAccount = await _resolveServiceAccountForUserResponse(microservice, transaction) + return res } @@ -2769,6 +3124,7 @@ module.exports = { updateMicroserviceEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceEndPoint), updateSystemMicroserviceEndPoint: TransactionDecorator.generateTransaction(updateSystemMicroserviceEndPoint), updateMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceConfigEndPoint), + updateMicroserviceCatalogEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceCatalogEndPoint), getMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(getMicroserviceConfigEndPoint), getSystemMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(getSystemMicroserviceConfigEndPoint), deleteMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(deleteMicroserviceConfigEndPoint), diff --git a/test/src/helpers/microservice-container-catalog.test.js b/test/src/helpers/microservice-container-catalog.test.js new file mode 100644 index 00000000..a38037e9 --- /dev/null +++ b/test/src/helpers/microservice-container-catalog.test.js @@ -0,0 +1,174 @@ +'use strict' + +const { expect } = require('chai') + +const CatalogContainer = require('../../../src/helpers/microservice-container-catalog') +const Errors = require('../../../src/helpers/errors') + +describe('Microservice container catalog helpers', () => { + describe('.catalogRequiresRebuild()', () => { + const populated = { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + + it('rebuilds empty to non-empty', () => { + expect(CatalogContainer.catalogRequiresRebuild(null, populated)).to.equal(true) + }) + + it('rebuilds non-empty to empty', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { items: [] })).to.equal(true) + }) + + it('rebuilds bindPath changes', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { + ...populated, + bindPath: '/opt/models' + })).to.equal(true) + }) + + it('rebuilds permissions changes', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { + ...populated, + permissions: 'rw' + })).to.equal(true) + }) + + it('does not rebuild item add/remove with the same bindPath and permissions', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }, { name: 'qwen3-8-27b' }] + })).to.equal(false) + }) + }) + + describe('.resolveProcessArgv()', () => { + it('prefers commands over cmd', () => { + expect(CatalogContainer.resolveProcessArgv({ + commands: ['a'], + cmd: ['b'] + })).to.eql(['a']) + }) + + it('accepts cmd as an alias', () => { + expect(CatalogContainer.resolveProcessArgv({ cmd: ['b'] })).to.eql(['b']) + }) + }) + + describe('.serializeArgv()', () => { + it('omits empty arrays', () => { + expect(CatalogContainer.serializeArgv([])).to.equal(null) + expect(CatalogContainer.serializeArgv(undefined)).to.equal(null) + }) + }) + + describe('.validateCatalog()', () => { + it('requires bindPath when items are present', () => { + expect(() => CatalogContainer.validateCatalog({ + items: [{ name: 'test-model' }] + })).to.throw(Errors.ValidationError) + }) + + it('rejects duplicate item names', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/models', + items: [{ name: 'test-model' }, { name: 'test-model' }] + })).to.throw(Errors.ValidationError) + }) + + it('rejects bindPath collisions with volumes', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/data', + items: [{ name: 'test-model' }] + }, { + volumeMappings: [{ containerDestination: '/data' }] + })).to.throw(Errors.ValidationError) + }) + }) + + describe('.validateContainerFields()', () => { + it('rejects runAsUser with colon when runAsGroup is set', () => { + expect(() => CatalogContainer.validateContainerFields({ + runAsUser: '1000:1000', + runAsGroup: '1000' + })).to.throw(Errors.ValidationError) + }) + + it('rejects memorySwap without memoryLimit unless -1', () => { + expect(() => CatalogContainer.validateContainerFields({ + memorySwap: 512 + })).to.throw(Errors.ValidationError) + expect(() => CatalogContainer.validateContainerFields({ + memorySwap: -1 + })).to.not.throw() + }) + + it('rejects unknown sysctls', () => { + expect(() => CatalogContainer.validateContainerFields({ + sysctls: { 'kernel.unprivileged_bpf_disabled': '1' } + })).to.throw(Errors.ValidationError) + }) + + it('coerces numeric sysctl values to strings', () => { + const data = { sysctls: { 'net.ipv4.tcp_syncookies': 1 } } + expect(() => CatalogContainer.validateContainerFields(data)).to.not.throw() + expect(data.sysctls['net.ipv4.tcp_syncookies']).to.equal('1') + }) + + it('rejects net sysctls when hostNetworkMode is true', () => { + expect(() => CatalogContainer.validateContainerFields({ + hostNetworkMode: true, + sysctls: { 'net.ipv4.tcp_syncookies': '1' } + })).to.throw(Errors.ValidationError) + }) + + it('rejects unknown ulimit keys and scalar values', () => { + expect(() => CatalogContainer.validateContainerFields({ + ulimits: { nofile: 1024 } + })).to.throw(Errors.ValidationError) + expect(() => CatalogContainer.validateContainerFields({ + ulimits: { as: { soft: 1, hard: 1 } } + })).to.throw(Errors.ValidationError) + }) + + it('rejects hostPath devices outside /dev', () => { + expect(() => CatalogContainer.validateContainerFields({ + devices: [{ hostPath: '/etc/passwd', containerPath: '/dev/foo' }] + })).to.throw(Errors.ValidationError) + }) + }) + + describe('.applyAgentContainerFields()', () => { + it('omits empty argv arrays and includes a non-empty catalog', () => { + const response = { uuid: 'ms' } + CatalogContainer.applyAgentContainerFields(response, { + sysctls: null + }, { + cmd: [], + entrypoint: [], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + }) + + expect(response).to.not.have.property('entrypoint') + expect(response).to.not.have.property('commands') + expect(response.cmd).to.eql([]) + expect(response.models.items).to.eql([{ name: 'test-model' }]) + }) + + it('emits commands from MicroserviceArgs rows', () => { + const response = { uuid: 'ms' } + CatalogContainer.applyAgentContainerFields(response, {}, { + cmd: ['python', 'app.py'] + }) + + expect(response.commands).to.eql(['python', 'app.py']) + expect(response.cmd).to.eql(['python', 'app.py']) + }) + }) +}) diff --git a/test/src/routes/microservices-routes.test.js b/test/src/routes/microservices-routes.test.js new file mode 100644 index 00000000..e6970b19 --- /dev/null +++ b/test/src/routes/microservices-routes.test.js @@ -0,0 +1,12 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Microservice routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/microservices.js'), 'utf8') + + it('registers catalog-only PATCH on a microservice', () => { + expect(source).to.include("path: '/api/v3/microservices/:uuid/models'") + expect(source).to.include('updateMicroserviceCatalogEndPoint') + }) +}) diff --git a/test/src/services/microservices-service.test.js b/test/src/services/microservices-service.test.js index c036f818..928e0512 100644 --- a/test/src/services/microservices-service.test.js +++ b/test/src/services/microservices-service.test.js @@ -18,12 +18,23 @@ const VolumeMappingManager = require('../../../src/data/managers/volume-mapping- const MicroserviceExtraHostManager = require('../../../src/data/managers/microservice-extra-host-manager') const MicroserviceEnvManager = require('../../../src/data/managers/microservice-env-manager') const MicroserviceArgManager = require('../../../src/data/managers/microservice-arg-manager') +const MicroserviceEntrypointManager = require('../../../src/data/managers/microservice-entrypoint-manager') +const MicroserviceDeviceManager = require('../../../src/data/managers/microservice-device-manager') +const MicroserviceTmpfsManager = require('../../../src/data/managers/microservice-tmpfs-manager') +const MicroserviceUlimitManager = require('../../../src/data/managers/microservice-ulimit-manager') +const MicroserviceModelManager = require('../../../src/data/managers/microservice-model-manager') +const MicroserviceModelItemManager = require('../../../src/data/managers/microservice-model-item-manager') const MicroserviceCdiDevManager = require('../../../src/data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../../../src/data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../../../src/data/managers/microservice-cap-drop-manager') const MicroserviceHealthCheckManager = require('../../../src/data/managers/microservice-healthcheck-manager') const VolumeMountService = require('../../../src/services/volume-mount-service') +const RuntimeClassService = require('../../../src/services/runtime-class-service') +const ModelService = require('../../../src/services/model-service') +const MicroserviceTemplateService = require('../../../src/services/microservice-template-service') +const FleetModelManager = require('../../../src/data/managers/fleet-model-manager') const RbacRoleManager = require('../../../src/data/managers/rbac-role-manager') +const ErrorMessages = require('../../../src/helpers/error-messages') const RbacServiceAccountManager = require('../../../src/data/managers/rbac-service-account-manager') const NatsAuthService = require('../../../src/services/nats-auth-service') const Errors = require('../../../src/helpers/errors') @@ -47,19 +58,45 @@ function buildMicroserviceRecord (fields = {}) { } } +function stubChildTableDeps (sandbox) { + sandbox.stub(MicroserviceArgManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceArgManager, 'findAllExcludeFields').resolves([]) + sandbox.stub(MicroserviceArgManager, 'create').resolves() + sandbox.stub(MicroserviceArgManager, 'delete').resolves() + sandbox.stub(MicroserviceEntrypointManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceEntrypointManager, 'create').resolves() + sandbox.stub(MicroserviceEntrypointManager, 'delete').resolves() + sandbox.stub(MicroserviceDeviceManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceDeviceManager, 'create').resolves() + sandbox.stub(MicroserviceDeviceManager, 'delete').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceTmpfsManager, 'create').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'delete').resolves() + sandbox.stub(MicroserviceUlimitManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceUlimitManager, 'create').resolves() + sandbox.stub(MicroserviceUlimitManager, 'delete').resolves() + sandbox.stub(MicroserviceModelManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceModelManager, 'create').resolves() + sandbox.stub(MicroserviceModelManager, 'delete').resolves() + sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceModelItemManager, 'create').resolves() + sandbox.stub(MicroserviceModelItemManager, 'delete').resolves() +} + function stubBuildGetResponseDeps (sandbox) { sandbox.stub(MicroservicePortService, 'getPortMappings').resolves([]) sandbox.stub(MicroserviceExtraHostManager, 'findAll').resolves([]) sandbox.stub(CatalogItemImageManager, 'findAll').resolves([]) sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) sandbox.stub(MicroserviceEnvManager, 'findAllExcludeFields').resolves([]) - sandbox.stub(MicroserviceArgManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceCdiDevManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceCapAddManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceCapDropManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceStatusManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceExecStatusManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceHealthCheckManager, 'findAllExcludeFields').resolves([]) + sandbox.stub(RbacServiceAccountManager, 'findOneByMicroserviceUuid').resolves(null) + stubChildTableDeps(sandbox) } function stubServiceAccountDeps (sandbox) { @@ -97,12 +134,14 @@ function stubCreateMicroserviceDeps (sandbox, { msvcUuid = 'msvc-uuid', appId = return Promise.resolve(null) }) sandbox.stub(MicroservicePortService, 'validatePortMappings').resolves() + sandbox.stub(MicroservicePortService, 'switchOnUpdateFlagsForMicroservicesForPortMapping').resolves() sandbox.stub(CatalogItemImageManager, 'bulkCreate').resolves() sandbox.stub(MicroserviceStatusManager, 'create').resolves() sandbox.stub(MicroserviceExecStatusManager, 'create').resolves() sandbox.stub(ChangeTrackingService, 'update').resolves() sandbox.stub(VolumeMappingManager, 'bulkCreate').resolves() stubServiceAccountDeps(sandbox) + stubChildTableDeps(sandbox) } function stubUpdateMicroserviceDeps (sandbox, existing) { @@ -112,6 +151,7 @@ function stubUpdateMicroserviceDeps (sandbox, existing) { sandbox.stub(Validator, 'validate').resolves(true) sandbox.stub(AppHelper, 'deleteUndefinedFields').callsFake((value) => value) sandbox.stub(FogManager, 'findOne').resolves(fog) + sandbox.stub(RegistryManager, 'findOne').resolves({ id: existing.registryId || 1, type: 'oci' }) sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves({ ...existing, catalogItem: existing.catalogItem || null, @@ -125,7 +165,9 @@ function stubUpdateMicroserviceDeps (sandbox, existing) { sandbox.stub(ServiceManager, 'findOne').resolves(null) sandbox.stub(ChangeTrackingService, 'update').resolves() sandbox.stub(VolumeMappingManager, 'delete').resolves() + sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) stubServiceAccountDeps(sandbox) + stubChildTableDeps(sandbox) } describe('Microservices Service', () => { @@ -215,6 +257,81 @@ describe('Microservices Service', () => { }) }) + describe('.buildGetMicroserviceResponse()', () => { + const msvcUuid = 'msvc-uuid' + const roleRef = { + kind: 'Role', + name: 'custom-role', + apiGroup: 'edgelet.iofog.org/v1' + } + + def('subject', () => $service.buildGetMicroserviceResponse($microservice, transaction)) + + beforeEach(() => { + stubBuildGetResponseDeps($sandbox) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ name: 'my-app' }) + }) + + context('when serviceAccount is preloaded on the microservice', () => { + def('microservice', () => ({ + uuid: msvcUuid, + name: 'test-msvc', + applicationId: 42, + logSize: 1024, + serviceAccount: { name: 'test-msvc', roleRef } + })) + + it('returns shaped serviceAccount with roleRef only', async () => { + const result = await $subject + expect(RbacServiceAccountManager.findOneByMicroserviceUuid).to.not.have.been.called + expect(result.serviceAccount).to.eql({ roleRef }) + }) + + it('does not leak raw serviceAccount database fields', async () => { + const result = await $subject + expect(result.serviceAccount).to.not.have.property('id') + expect(result.serviceAccount).to.not.have.property('microserviceUuid') + expect(result.serviceAccount).to.not.have.property('roleId') + }) + }) + + context('when serviceAccount is not preloaded', () => { + def('microservice', () => ({ + uuid: msvcUuid, + name: 'test-msvc', + applicationId: 42, + logSize: 1024 + })) + + beforeEach(() => { + RbacServiceAccountManager.findOneByMicroserviceUuid.resolves({ + name: 'test-msvc', + roleRef + }) + }) + + it('loads serviceAccount by microservice uuid', async () => { + const result = await $subject + expect(RbacServiceAccountManager.findOneByMicroserviceUuid).to.have.been.calledWith(msvcUuid, transaction) + expect(result.serviceAccount).to.eql({ roleRef }) + }) + }) + + context('when microservice has no service account', () => { + def('microservice', () => ({ + uuid: msvcUuid, + name: 'test-msvc', + applicationId: 42, + logSize: 1024 + })) + + it('returns null serviceAccount', async () => { + const result = await $subject + expect(result.serviceAccount).to.equal(null) + }) + }) + }) + describe('.createMicroserviceEndPoint()', () => { const msvcUuid = 'new-msvc-uuid' const microserviceData = { @@ -287,6 +404,60 @@ describe('Microservices Service', () => { expect(VolumeMappingManager.create).to.have.been.called }) }) + + context('when runtime is set', () => { + it('auto-links the runtime class on an edgelet fog', async () => { + FogManager.findOne.resolves({ + uuid: 'fog-uuid', + archId: 1, + name: 'edge-1', + availableRuntimes: ['spin'], + containerEngine: 'edgelet' + }) + const ensure = $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog').resolves(true) + + await $service.createMicroserviceEndPoint({ + ...microserviceData, + runtime: 'spin' + }, isCLI, transaction) + + expect(ensure).to.have.been.calledWith('fog-uuid', 'spin', transaction) + }) + + it('rejects when the runtime is not in availableRuntimes', async () => { + FogManager.findOne.resolves({ + uuid: 'fog-uuid', + archId: 1, + name: 'edge-1', + availableRuntimes: ['runc'], + containerEngine: 'edgelet' + }) + const ensure = $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog').resolves(true) + + await expect($service.createMicroserviceEndPoint({ + ...microserviceData, + runtime: 'spin' + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /not available/) + expect(ensure).to.not.have.been.called + }) + + it('rejects when the runtime class row is missing', async () => { + FogManager.findOne.resolves({ + uuid: 'fog-uuid', + archId: 1, + name: 'edge-1', + availableRuntimes: ['spin'], + containerEngine: 'edgelet' + }) + $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog') + .rejects(new Errors.ValidationError("RuntimeClass 'spin' does not exist")) + + await expect($service.createMicroserviceEndPoint({ + ...microserviceData, + runtime: 'spin' + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /does not exist/) + }) + }) }) describe('.updateMicroserviceEndPoint()', () => { @@ -310,12 +481,88 @@ describe('Microservices Service', () => { expect(ChangeTrackingService.update).to.have.been.called }) + context('when runtime is set', () => { + def('updateData', () => ({ runtime: 'spin' })) + + it('auto-links the runtime class on an edgelet fog', async () => { + FogManager.findOne.resolves({ + uuid: existing.iofogUuid, + archId: 1, + name: 'edge-1', + availableRuntimes: ['spin'], + containerEngine: 'edgelet' + }) + const ensure = $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog').resolves(true) + + await $subject + expect(ensure).to.have.been.calledWith(existing.iofogUuid, 'spin', transaction) + }) + }) + context('when renaming', () => { def('updateData', () => ({ name: 'new-name' })) it('rejects rename attempts', () => expect($subject).to.be.rejectedWith('Microservice Resource Name is immutable')) }) + context('when rebasing from a template', () => { + def('updateData', () => ({ + template: { + name: 'nginx-edge', + variables: { port: '9090' } + } + })) + + beforeEach(() => { + MicroserviceManager.findOne.callsFake((where) => { + if (where && where.name && where.uuid) { + return Promise.resolve(null) + } + return Promise.resolve(existing) + }) + $sandbox.stub(CatalogItemImageManager, 'delete').resolves() + $sandbox.stub(CatalogItemImageManager, 'bulkCreate').resolves() + $sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + $sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves(['test-model']) + $sandbox.stub(MicroserviceTemplateService, 'getMicroserviceDataFromTemplate').resolves({ + images: [{ containerImage: 'nginx:alpine', archId: 1 }], + registryId: 1, + commands: ['nginx', '-g', 'daemon off;'], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + }) + }) + + it('expands the template and preserves identity from the existing microservice', async () => { + await $subject + + expect(MicroserviceTemplateService.getMicroserviceDataFromTemplate).to.have.been.calledWith( + { name: 'nginx-edge', variables: { port: '9090' } }, + isCLI, + transaction + ) + expect(Validator.validate).to.have.been.calledWith( + sinon.match({ + name: existing.name, + iofogUuid: existing.iofogUuid, + images: [{ containerImage: 'nginx:alpine', archId: 1 }] + }), + Validator.schemas.microserviceUpdate + ) + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'nginx' }), + transaction + ) + }) + }) + context('when microservice is controller', () => { beforeEach(() => { MicroserviceManager.findOneWithCategory.resolves({ @@ -641,4 +888,272 @@ describe('Microservices Service', () => { }) }) }) + + describe('.createMicroserviceEndPoint() catalog and registry', () => { + const msvcUuid = 'new-msvc-uuid' + const microserviceData = { + name: 'new-msvc', + application: 'my-app', + iofogUuid: 'fog-uuid', + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1 + } + + beforeEach(() => { + stubCreateMicroserviceDeps($sandbox, { msvcUuid }) + }) + + it('rejects a Hugging Face registry for microservice images', async () => { + RegistryManager.findOne.resolves({ id: 3, type: 'hf' }) + + return expect($service.createMicroserviceEndPoint({ + ...microserviceData, + registryId: 3 + }, isCLI, transaction)).to.be.rejectedWith( + AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 3) + ) + }) + + it('auto-attaches catalog model names to the target agent', async () => { + $sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + const attach = $sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves(['test-model']) + + await $service.createMicroserviceEndPoint({ + ...microserviceData, + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + }, isCLI, transaction) + + expect(attach).to.have.been.calledWith('fog-uuid', ['test-model'], transaction) + expect(MicroserviceModelManager.create).to.have.been.calledWith( + sinon.match({ bindPath: '/models', permissions: 'ro' }), + transaction + ) + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + const created = MicroserviceManager.create.firstCall.args[0] + expect(created).to.not.have.property('models') + expect(created).to.not.have.property('commands') + }) + + it('persists commands when cmd is sent as an alias', async () => { + await $service.createMicroserviceEndPoint({ + ...microserviceData, + cmd: ['python', 'app.py'] + }, isCLI, transaction) + + const created = MicroserviceManager.create.firstCall.args[0] + expect(created).to.not.have.property('commands') + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'python' }), + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'app.py' }), + transaction + ) + }) + + it('deploys from a template with identity overlay and merged catalog/container fields', async () => { + $sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + $sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves(['test-model']) + $sandbox.stub(MicroserviceTemplateService, 'getMicroserviceDataFromTemplate').resolves({ + images: [{ containerImage: 'nginx:alpine', archId: 1 }], + registryId: 1, + commands: ['nginx', '-g', 'daemon off;'], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + }) + + await $service.createMicroserviceEndPoint({ + name: 'my-instance', + application: 'my-app', + iofogUuid: 'fog-uuid', + template: { + name: 'nginx-edge', + variables: { port: '8080' } + } + }, isCLI, transaction) + + expect(MicroserviceTemplateService.getMicroserviceDataFromTemplate).to.have.been.calledWith( + { name: 'nginx-edge', variables: { port: '8080' } }, + isCLI, + transaction + ) + expect(Validator.validate).to.have.been.calledWith( + sinon.match({ + name: 'my-instance', + application: 'my-app', + iofogUuid: 'fog-uuid', + images: [{ containerImage: 'nginx:alpine', archId: 1 }] + }), + Validator.schemas.microserviceCreate + ) + const created = MicroserviceManager.create.firstCall.args[0] + expect(created.name).to.equal('my-instance') + expect(created).to.not.have.property('models') + expect(created).to.not.have.property('commands') + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'nginx' }), + transaction + ) + }) + }) + + describe('.updateMicroserviceCatalogEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const existingCatalog = { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + + function stubCatalogPatch (sandbox, existing, previousCatalog = null) { + sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(existing) + sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) + sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves([]) + sandbox.stub(MicroserviceManager, 'update').resolves() + sandbox.stub(ChangeTrackingService, 'update').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceModelManager, 'findOne').resolves( + previousCatalog && previousCatalog.bindPath + ? { bindPath: previousCatalog.bindPath, permissions: previousCatalog.permissions || 'ro' } + : null + ) + sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves( + previousCatalog && Array.isArray(previousCatalog.items) + ? previousCatalog.items.map((item, id) => ({ id, name: item.name })) + : [] + ) + sandbox.stub(MicroserviceModelManager, 'delete').resolves() + sandbox.stub(MicroserviceModelManager, 'create').resolves() + sandbox.stub(MicroserviceModelItemManager, 'delete').resolves() + sandbox.stub(MicroserviceModelItemManager, 'create').resolves() + } + + it('rebuilds when the catalog goes from empty to non-empty', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + items: [{ name: 'test-model' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceModels, + transaction + ) + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + }) + + it('rebuilds when the catalog goes from non-empty to empty', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + items: [] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when bindPath changes', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/opt/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when permissions change', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + permissions: 'rw', + items: [{ name: 'test-model' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + }) + + it('does not rebuild when only items change on a non-empty catalog', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + FleetModelManager.findOne.callsFake(async (where) => ({ uuid: 'model-uuid', name: where.name })) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }, { name: 'qwen3-8-27b' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: false }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceModels, + transaction + ) + }) + }) }) From 44ba58630e6c386c404c4786153408e96aa459b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:51:00 +0300 Subject: [PATCH 08/55] Add microservice templates with YAML and deploy overlay. Create accepts a template name and variables so request fields win over the stored spec. --- .../microservice-template-controller.js | 67 +++++ .../managers/microservice-template-manager.js | 58 ++++ .../microservice-template-variable-manager.js | 12 + .../models/microservice-template-variable.js | 43 +++ src/data/models/microservice-template.js | 45 +++ src/routes/microserviceTemplates.js | 142 +++++++++ src/schemas/microservice-template.js | 98 ++++++ src/services/microservice-template-service.js | 278 ++++++++++++++++++ .../microserviceTemplates-routes.test.js | 18 ++ .../microservice-template-service.test.js | 230 +++++++++++++++ 10 files changed, 991 insertions(+) create mode 100644 src/controllers/microservice-template-controller.js create mode 100644 src/data/managers/microservice-template-manager.js create mode 100644 src/data/managers/microservice-template-variable-manager.js create mode 100644 src/data/models/microservice-template-variable.js create mode 100644 src/data/models/microservice-template.js create mode 100644 src/routes/microserviceTemplates.js create mode 100644 src/schemas/microservice-template.js create mode 100644 src/services/microservice-template-service.js create mode 100644 test/src/routes/microserviceTemplates-routes.test.js create mode 100644 test/src/services/microservice-template-service.test.js diff --git a/src/controllers/microservice-template-controller.js b/src/controllers/microservice-template-controller.js new file mode 100644 index 00000000..6ad39621 --- /dev/null +++ b/src/controllers/microservice-template-controller.js @@ -0,0 +1,67 @@ +const MicroserviceTemplateService = require('../services/microservice-template-service') +const YAMLParserService = require('../services/yaml-parser-service') +const errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const { rvaluesVarSubstition } = require('../helpers/template-helper') + +async function _applyVariableSelfSubstitution (templateData) { + if (!templateData.variables) { + return templateData + } + await rvaluesVarSubstition(templateData.variables, { self: templateData.variables }) + return templateData +} + +const listMicroserviceTemplatesEndpoint = async (req) => { + return MicroserviceTemplateService.listMicroserviceTemplatesEndpoint() +} + +const getMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.getMicroserviceTemplateEndpoint(req.params.name) +} + +const createMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.createMicroserviceTemplateEndpoint(req.body) +} + +const updateMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.updateMicroserviceTemplateEndpoint(req.params.name, req.body) +} + +const deleteMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.deleteMicroserviceTemplateEndpoint(req.params.name) +} + +const createMicroserviceTemplateYamlEndpoint = async (req) => { + if (!req.file) { + throw new errors.ValidationError(ErrorMessages.MICROSERVICE_TEMPLATE_FILE_NOT_FOUND) + } + const fileContent = req.file.buffer.toString() + const templateData = await YAMLParserService.parseMicroserviceTemplateFile(fileContent) + await _applyVariableSelfSubstitution(templateData) + return MicroserviceTemplateService.createMicroserviceTemplateEndpoint(templateData) +} + +const upsertMicroserviceTemplateYamlEndpoint = async (req) => { + if (!req.file) { + throw new errors.ValidationError(ErrorMessages.MICROSERVICE_TEMPLATE_FILE_NOT_FOUND) + } + const name = req.params.name + const fileContent = req.file.buffer.toString() + const templateData = await YAMLParserService.parseMicroserviceTemplateFile(fileContent, { + isUpdate: true, + templateName: name + }) + await _applyVariableSelfSubstitution(templateData) + return MicroserviceTemplateService.upsertMicroserviceTemplateEndpoint(name, templateData) +} + +module.exports = { + listMicroserviceTemplatesEndpoint, + getMicroserviceTemplateEndpoint, + createMicroserviceTemplateEndpoint, + updateMicroserviceTemplateEndpoint, + deleteMicroserviceTemplateEndpoint, + createMicroserviceTemplateYamlEndpoint, + upsertMicroserviceTemplateYamlEndpoint +} diff --git a/src/data/managers/microservice-template-manager.js b/src/data/managers/microservice-template-manager.js new file mode 100644 index 00000000..34044212 --- /dev/null +++ b/src/data/managers/microservice-template-manager.js @@ -0,0 +1,58 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceTemplate = models.MicroserviceTemplate + +const excludedFields = [ + 'created_at', + 'updated_at' +] + +const variablesInclude = { + model: models.MicroserviceTemplateVariable, + as: 'variables', + required: false, + attributes: { exclude: excludedFields } +} + +class MicroserviceTemplateManager extends BaseManager { + getEntity () { + return MicroserviceTemplate + } + + findOne (where, transaction) { + return MicroserviceTemplate.findOne({ + where, + attributes: { exclude: excludedFields }, + transaction + }) + } + + findAll (where, transaction) { + return MicroserviceTemplate.findAll({ + where, + attributes: { exclude: excludedFields }, + transaction + }) + } + + findOnePopulated (where, transaction) { + return MicroserviceTemplate.findOne({ + include: [variablesInclude], + where, + attributes: { exclude: excludedFields }, + transaction + }) + } + + findAllPopulated (where, transaction) { + return MicroserviceTemplate.findAll({ + include: [variablesInclude], + where, + attributes: { exclude: excludedFields }, + transaction + }) + } +} + +const instance = new MicroserviceTemplateManager() +module.exports = instance diff --git a/src/data/managers/microservice-template-variable-manager.js b/src/data/managers/microservice-template-variable-manager.js new file mode 100644 index 00000000..d775f532 --- /dev/null +++ b/src/data/managers/microservice-template-variable-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceTemplateVariable = models.MicroserviceTemplateVariable + +class MicroserviceTemplateVariableManager extends BaseManager { + getEntity () { + return MicroserviceTemplateVariable + } +} + +const instance = new MicroserviceTemplateVariableManager() +module.exports = instance diff --git a/src/data/models/microservice-template-variable.js b/src/data/models/microservice-template-variable.js new file mode 100644 index 00000000..d317fb24 --- /dev/null +++ b/src/data/models/microservice-template-variable.js @@ -0,0 +1,43 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const MicroserviceTemplateVariable = sequelize.define('MicroserviceTemplateVariable', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + key: { + type: DataTypes.TEXT, + field: 'key' + }, + description: { + type: DataTypes.TEXT, + field: 'description', + defaultValue: '' + }, + defaultValue: { + type: DataTypes.TEXT, + field: 'default_value' + } + }, { + tableName: 'MicroserviceTemplateVariables', + timestamps: true, + underscored: true + }) + + MicroserviceTemplateVariable.associate = function (models) { + MicroserviceTemplateVariable.belongsTo(models.MicroserviceTemplate, { + foreignKey: { + name: 'microserviceTemplateId', + field: 'microservice_template_id' + }, + as: 'microserviceTemplate', + onDelete: 'cascade' + }) + } + + return MicroserviceTemplateVariable +} diff --git a/src/data/models/microservice-template.js b/src/data/models/microservice-template.js new file mode 100644 index 00000000..f53db1fa --- /dev/null +++ b/src/data/models/microservice-template.js @@ -0,0 +1,45 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const MicroserviceTemplate = sequelize.define('MicroserviceTemplate', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + field: 'name' + }, + description: { + type: DataTypes.TEXT, + field: 'description', + defaultValue: '' + }, + microserviceJSON: { + type: DataTypes.TEXT, + field: 'microservice_json', + defaultValue: '{}' + } + }, { + tableName: 'MicroserviceTemplates', + timestamps: true, + underscored: true + }) + + MicroserviceTemplate.associate = function (models) { + MicroserviceTemplate.hasMany(models.MicroserviceTemplateVariable, { + foreignKey: { + name: 'microserviceTemplateId', + field: 'microservice_template_id' + }, + as: 'variables' + }) + } + + return MicroserviceTemplate +} diff --git a/src/routes/microserviceTemplates.js b/src/routes/microserviceTemplates.js new file mode 100644 index 00000000..2822478b --- /dev/null +++ b/src/routes/microserviceTemplates.js @@ -0,0 +1,142 @@ +const constants = require('../helpers/constants') +const MicroserviceTemplateController = require('../controllers/microservice-template-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/microserviceTemplates', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.listMicroserviceTemplatesEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/microserviceTemplates/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.getMicroserviceTemplateEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/microserviceTemplates/:name', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.updateMicroserviceTemplateEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/microserviceTemplates/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.deleteMicroserviceTemplateEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/microserviceTemplates', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.createMicroserviceTemplateEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/microserviceTemplates/yaml', + fileInput: 'template', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.createMicroserviceTemplateYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/microserviceTemplates/yaml/:name', + fileInput: 'template', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.upsertMicroserviceTemplateYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/schemas/microservice-template.js b/src/schemas/microservice-template.js new file mode 100644 index 00000000..154e3c8d --- /dev/null +++ b/src/schemas/microservice-template.js @@ -0,0 +1,98 @@ +const { nameRegex, serviceNameRegex } = require('./utils/utils') + +const microserviceTemplateVariable = { + id: '/microserviceTemplateVariable', + type: 'object', + properties: { + key: { + type: 'string', + minLength: 1, + pattern: nameRegex + }, + description: { type: 'string' } + }, + additionalProperties: true, + required: ['key'] +} + +const microserviceTemplateCreate = { + id: '/microserviceTemplateCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + description: { type: 'string' }, + variables: { + type: 'array', + items: { $ref: '/microserviceTemplateVariable' } + }, + microservice: { + type: 'object' + } + }, + required: ['name', 'microservice'], + additionalProperties: false +} + +const microserviceTemplateUpdate = { + id: '/microserviceTemplateUpdate', + type: 'object', + properties: { + description: { type: 'string' }, + variables: { + type: 'array', + items: { $ref: '/microserviceTemplateVariable' } + }, + microservice: { + type: 'object' + } + }, + additionalProperties: false +} + +const microserviceTemplateDeploy = { + id: '/microserviceTemplateDeploy', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + variables: { + oneOf: [ + { + type: 'object' + }, + { + type: 'array', + items: { + type: 'object', + properties: { + key: { type: 'string' }, + value: {} + }, + required: ['key'] + } + } + ] + } + }, + required: ['name'], + additionalProperties: true +} + +module.exports = { + mainSchemas: [ + microserviceTemplateCreate, + microserviceTemplateVariable, + microserviceTemplateUpdate, + microserviceTemplateDeploy + ], + innerSchemas: [microserviceTemplateVariable] +} diff --git a/src/services/microservice-template-service.js b/src/services/microservice-template-service.js new file mode 100644 index 00000000..de69d997 --- /dev/null +++ b/src/services/microservice-template-service.js @@ -0,0 +1,278 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const MicroserviceTemplateManager = require('../data/managers/microservice-template-manager') +const MicroserviceTemplateVariableManager = require('../data/managers/microservice-template-variable-manager') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') + +const IDENTITY_FIELDS = ['name', 'application', 'iofogUuid', 'agentName', 'flowId', 'template'] + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function _cloneJson (value, fallback) { + if (value == null || value === '') { + return fallback + } + if (typeof value === 'object') { + return JSON.parse(JSON.stringify(value)) + } + try { + return JSON.parse(value) + } catch (error) { + return fallback + } +} + +function _parseStoredDefaultValue (defaultValue) { + if (defaultValue === undefined) { + return undefined + } + if (typeof defaultValue !== 'string') { + return defaultValue + } + try { + return JSON.parse(defaultValue) + } catch (error) { + return defaultValue + } +} + +function stripIdentityFields (microservice) { + const clone = { ...(microservice || {}) } + for (const field of IDENTITY_FIELDS) { + delete clone[field] + } + return clone +} + +function toWireVariable (variable) { + const plain = _toPlain(variable) || variable + const wire = { + key: plain.key + } + if (plain.description !== undefined) { + wire.description = plain.description + } + if (plain.defaultValue !== undefined) { + wire.defaultValue = _parseStoredDefaultValue(plain.defaultValue) + } + return wire +} + +function toWireTemplate (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + return { + name: plain.name, + description: plain.description != null ? plain.description : '', + variables: (plain.variables || []).map(toWireVariable), + microservice: _cloneJson(plain.microserviceJSON, {}) + } +} + +function _buildUserVariableMap (userVariables) { + if (Array.isArray(userVariables)) { + return userVariables.reduce((acc, variable) => { + if (variable && variable.key != null) { + acc[variable.key] = variable.value + } + return acc + }, {}) + } + if (userVariables && typeof userVariables === 'object') { + return { ...userVariables } + } + return {} +} + +function _buildDefaultVariableMap (schemaVariables) { + return (schemaVariables || []).reduce((acc, variable) => { + if (variable && variable.key != null && variable.defaultValue !== undefined) { + acc[variable.key] = variable.defaultValue + } + return acc + }, {}) +} + +function _parseJsonObjectField (value) { + if (typeof value !== 'string' || value === '') { + return undefined + } + const trimmed = value.trim() + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) { + return undefined + } + try { + const parsed = JSON.parse(value) + if (parsed && typeof parsed === 'object') { + return parsed + } + } catch (error) { + return undefined + } + return undefined +} + +async function _createVariable (microserviceTemplateId, variableData, transaction) { + const newVariable = { + ...variableData + } + if (newVariable.defaultValue !== undefined) { + newVariable.defaultValue = JSON.stringify(newVariable.defaultValue) + } + return MicroserviceTemplateVariableManager.create({ + ...newVariable, + microserviceTemplateId + }, transaction) +} + +async function _updateVariables (microserviceTemplateId, variables, transaction) { + await MicroserviceTemplateVariableManager.delete({ microserviceTemplateId }, transaction) + for (const variableData of variables || []) { + await _createVariable(microserviceTemplateId, variableData, transaction) + } +} + +async function getTemplateByName (name, transaction) { + const template = await MicroserviceTemplateManager.findOnePopulated({ name }, transaction) + if (!template) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_NOT_FOUND, name)) + } + return template +} + +async function listMicroserviceTemplatesEndpoint (transaction) { + const templates = await MicroserviceTemplateManager.findAllPopulated({}, transaction) + return { + microserviceTemplates: templates.map(toWireTemplate) + } +} + +async function getMicroserviceTemplateEndpoint (name, transaction) { + const template = await getTemplateByName(name, transaction) + return toWireTemplate(template) +} + +async function createMicroserviceTemplateEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.microserviceTemplateCreate) + + const existing = await MicroserviceTemplateManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_ALREADY_EXISTS, data.name)) + } + + const created = await MicroserviceTemplateManager.create({ + name: data.name, + description: data.description != null ? data.description : '', + microserviceJSON: JSON.stringify(stripIdentityFields(data.microservice)) + }, transaction) + + try { + if (data.variables) { + for (const variableData of data.variables) { + await _createVariable(created.id, variableData, transaction) + } + } + return getMicroserviceTemplateEndpoint(created.name, transaction) + } catch (error) { + await deleteMicroserviceTemplateEndpoint(created.name, transaction) + throw error + } +} + +async function updateMicroserviceTemplateEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.microserviceTemplateUpdate) + const existing = await getTemplateByName(name, transaction) + const current = toWireTemplate(existing) + + const nextMicroservice = data.microservice !== undefined + ? stripIdentityFields(data.microservice) + : current.microservice + const nextDescription = data.description !== undefined ? data.description : current.description + + await MicroserviceTemplateManager.update({ name }, { + description: nextDescription, + microserviceJSON: JSON.stringify(nextMicroservice) + }, transaction) + + if (data.variables !== undefined) { + await _updateVariables(existing.id, data.variables, transaction) + } + + return getMicroserviceTemplateEndpoint(name, transaction) +} + +async function upsertMicroserviceTemplateEndpoint (name, data, transaction) { + const existing = await MicroserviceTemplateManager.findOne({ name }, transaction) + if (existing) { + return updateMicroserviceTemplateEndpoint(name, data, transaction) + } + return createMicroserviceTemplateEndpoint({ ...data, name }, transaction) +} + +async function deleteMicroserviceTemplateEndpoint (name, transaction) { + await getTemplateByName(name, transaction) + await MicroserviceTemplateManager.delete({ name }, transaction) + return {} +} + +async function getMicroserviceDataFromTemplate (deploymentData, isCLI, transaction) { + await Validator.validate(deploymentData, Validator.schemas.microserviceTemplateDeploy) + + const templateRow = await MicroserviceTemplateManager.findOnePopulated({ name: deploymentData.name }, transaction) + if (!templateRow) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_NOT_FOUND, deploymentData.name)) + } + + const template = toWireTemplate(templateRow) + if (!template.microservice || typeof template.microservice !== 'object' || Array.isArray(template.microservice)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_INVALID, deploymentData.name)) + } + + const spec = stripIdentityFields(template.microservice) + const parsedConfig = _parseJsonObjectField(spec.config) + if (parsedConfig !== undefined) { + spec.config = parsedConfig + } + const parsedAnnotations = _parseJsonObjectField(spec.annotations) + if (parsedAnnotations !== undefined) { + spec.annotations = parsedAnnotations + } + + const { rvaluesVarSubstition } = require('../helpers/template-helper') + const variableMap = { + ..._buildDefaultVariableMap(template.variables), + ..._buildUserVariableMap(deploymentData.variables), + self: spec + } + await rvaluesVarSubstition(spec, variableMap) + + if (spec.config && typeof spec.config === 'object' && !Array.isArray(spec.config)) { + spec.config = JSON.stringify(spec.config) + } + if (spec.annotations && typeof spec.annotations === 'object' && !Array.isArray(spec.annotations)) { + spec.annotations = JSON.stringify(spec.annotations) + } + + return spec +} + +module.exports = { + listMicroserviceTemplatesEndpoint: TransactionDecorator.generateTransaction(listMicroserviceTemplatesEndpoint), + getMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(getMicroserviceTemplateEndpoint), + createMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(createMicroserviceTemplateEndpoint), + updateMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(updateMicroserviceTemplateEndpoint), + upsertMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(upsertMicroserviceTemplateEndpoint), + deleteMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(deleteMicroserviceTemplateEndpoint), + getMicroserviceDataFromTemplate, + toWireTemplate, + stripIdentityFields +} diff --git a/test/src/routes/microserviceTemplates-routes.test.js b/test/src/routes/microserviceTemplates-routes.test.js new file mode 100644 index 00000000..0cdcf11e --- /dev/null +++ b/test/src/routes/microserviceTemplates-routes.test.js @@ -0,0 +1,18 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Microservice template routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/microserviceTemplates.js'), 'utf8') + + it('registers template CRUD and YAML routes', () => { + expect(source).to.include("path: '/api/v3/microserviceTemplates'") + expect(source).to.include("path: '/api/v3/microserviceTemplates/:name'") + expect(source).to.include("path: '/api/v3/microserviceTemplates/yaml'") + expect(source).to.include("path: '/api/v3/microserviceTemplates/yaml/:name'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + }) +}) diff --git a/test/src/services/microservice-template-service.test.js b/test/src/services/microservice-template-service.test.js new file mode 100644 index 00000000..51fa9a15 --- /dev/null +++ b/test/src/services/microservice-template-service.test.js @@ -0,0 +1,230 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const MicroserviceTemplateManager = require('../../../src/data/managers/microservice-template-manager') +const MicroserviceTemplateVariableManager = require('../../../src/data/managers/microservice-template-variable-manager') +const MicroserviceTemplateService = require('../../../src/services/microservice-template-service') +const Errors = require('../../../src/helpers/errors') + +describe('Microservice Template Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => { + $sandbox.restore() + }) + + function templateSpec () { + return { + images: [{ containerImage: 'nginx:alpine', archId: 1 }], + registryId: 1, + commands: ['nginx', '-g', 'daemon off;'], + env: [{ key: 'PORT', value: '{{ port }}' }], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + } + } + + function stubTemplateRow (overrides = {}) { + return { + id: 1, + name: 'nginx-edge', + description: 'nginx at the edge', + microserviceJSON: JSON.stringify(templateSpec()), + variables: [ + { key: 'port', description: 'listen port', defaultValue: JSON.stringify('80') } + ], + ...overrides + } + } + + describe('.createMicroserviceTemplateEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findOne').resolves(null) + $sandbox.stub(MicroserviceTemplateManager, 'create').callsFake(async (data) => ({ id: 1, ...data })) + $sandbox.stub(MicroserviceTemplateVariableManager, 'create').resolves() + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(stubTemplateRow()) + }) + + it('creates a template and strips instance identity fields', async () => { + const result = await MicroserviceTemplateService.createMicroserviceTemplateEndpoint({ + name: 'nginx-edge', + description: 'nginx at the edge', + variables: [{ key: 'port', defaultValue: '80' }], + microservice: { + ...templateSpec(), + name: 'should-not-store', + application: 'should-not-store', + iofogUuid: 'fog-uuid' + } + }, transaction) + + expect(result.name).to.equal('nginx-edge') + expect(result.microservice).to.not.have.property('name') + expect(result.microservice).to.not.have.property('application') + expect(result.microservice).to.not.have.property('iofogUuid') + expect(result.microservice.models.items).to.eql([{ name: 'test-model' }]) + expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) + expect(result.variables[0].key).to.equal('port') + expect(result.variables[0].defaultValue).to.equal('80') + expect(MicroserviceTemplateManager.create).to.have.been.calledOnce + expect(MicroserviceTemplateVariableManager.create).to.have.been.calledWith( + sinon.match({ + key: 'port', + defaultValue: JSON.stringify('80'), + microserviceTemplateId: 1 + }), + transaction + ) + }) + + it('rejects a duplicate template name', async () => { + MicroserviceTemplateManager.findOne.resolves(stubTemplateRow()) + + await expect(MicroserviceTemplateService.createMicroserviceTemplateEndpoint({ + name: 'nginx-edge', + microservice: templateSpec() + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + }) + + describe('.listMicroserviceTemplatesEndpoint() and .getMicroserviceTemplateEndpoint()', () => { + const row = stubTemplateRow() + + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findAllPopulated').resolves([row]) + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(row) + }) + + it('lists templates with parsed microservice and variables', async () => { + const result = await MicroserviceTemplateService.listMicroserviceTemplatesEndpoint(transaction) + expect(result.microserviceTemplates).to.have.length(1) + expect(result.microserviceTemplates[0].name).to.equal('nginx-edge') + expect(result.microserviceTemplates[0].microservice.models.bindPath).to.equal('/models') + expect(result.microserviceTemplates[0].variables[0].defaultValue).to.equal('80') + }) + + it('returns a template by name', async () => { + const result = await MicroserviceTemplateService.getMicroserviceTemplateEndpoint('nginx-edge', transaction) + expect(result.name).to.equal('nginx-edge') + expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) + }) + }) + + describe('.updateMicroserviceTemplateEndpoint() and .deleteMicroserviceTemplateEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(stubTemplateRow()) + $sandbox.stub(MicroserviceTemplateManager, 'update').resolves() + $sandbox.stub(MicroserviceTemplateManager, 'delete').resolves() + $sandbox.stub(MicroserviceTemplateVariableManager, 'delete').resolves() + $sandbox.stub(MicroserviceTemplateVariableManager, 'create').resolves() + }) + + it('patches description and microservice spec', async () => { + MicroserviceTemplateManager.findOnePopulated.onSecondCall().resolves(stubTemplateRow({ + description: 'updated', + microserviceJSON: JSON.stringify({ + images: [{ containerImage: 'nginx:stable', archId: 1 }], + registryId: 1 + }) + })) + + const result = await MicroserviceTemplateService.updateMicroserviceTemplateEndpoint('nginx-edge', { + description: 'updated', + microservice: { + images: [{ containerImage: 'nginx:stable', archId: 1 }], + registryId: 1 + } + }, transaction) + + expect(MicroserviceTemplateManager.update).to.have.been.calledOnce + expect(MicroserviceTemplateVariableManager.delete).to.not.have.been.called + expect(result.description).to.equal('updated') + expect(result.microservice.images[0].containerImage).to.equal('nginx:stable') + }) + + it('replaces variables when they are sent on update', async () => { + MicroserviceTemplateManager.findOnePopulated.onSecondCall().resolves(stubTemplateRow({ + variables: [ + { key: 'port', defaultValue: JSON.stringify('8080') } + ] + })) + + const result = await MicroserviceTemplateService.updateMicroserviceTemplateEndpoint('nginx-edge', { + variables: [{ key: 'port', defaultValue: '8080' }] + }, transaction) + + expect(MicroserviceTemplateVariableManager.delete).to.have.been.calledWith( + { microserviceTemplateId: 1 }, + transaction + ) + expect(MicroserviceTemplateVariableManager.create).to.have.been.calledOnce + expect(result.variables[0].defaultValue).to.equal('8080') + }) + + it('deletes a template by name', async () => { + const result = await MicroserviceTemplateService.deleteMicroserviceTemplateEndpoint('nginx-edge', transaction) + expect(result).to.eql({}) + expect(MicroserviceTemplateManager.delete).to.have.been.calledWith({ name: 'nginx-edge' }, transaction) + }) + + it('rejects get/update/delete when the template is missing', async () => { + MicroserviceTemplateManager.findOnePopulated.resolves(null) + await expect(MicroserviceTemplateService.getMicroserviceTemplateEndpoint('missing', transaction)) + .to.be.rejectedWith(Errors.NotFoundError, /not found/) + }) + }) + + describe('.getMicroserviceDataFromTemplate()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(stubTemplateRow()) + }) + + it('substitutes variables and keeps catalog and container fields', async () => { + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge', + variables: { port: '8080' } + }, false, transaction) + + expect(result.env).to.eql([{ key: 'PORT', value: '8080' }]) + expect(result.commands).to.eql(['nginx', '-g', 'daemon off;']) + expect(result.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(result).to.not.have.property('name') + expect(result).to.not.have.property('application') + expect(result).to.not.have.property('iofogUuid') + }) + + it('uses default variable values when the deploy overlay omits them', async () => { + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge' + }, false, transaction) + + expect(result.env).to.eql([{ key: 'PORT', value: '80' }]) + }) + + it('accepts application-style array variables', async () => { + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge', + variables: [{ key: 'port', value: '9090' }] + }, false, transaction) + + expect(result.env[0].value).to.equal('9090') + }) + + it('rejects a missing template name', async () => { + MicroserviceTemplateManager.findOnePopulated.resolves(null) + await expect(MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'missing' + }, false, transaction)).to.be.rejectedWith(Errors.NotFoundError, /not found/) + }) + }) +}) From d2b51123e746c69ab24d4b0160e24f79fa6c0c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:51:07 +0300 Subject: [PATCH 09/55] Remove HAL and BLE provisioning while keeping Edge Guard. Drop hardware inventory routes, RESTBlue/HAL catalog items, and fog bluetooth/hal flags. --- src/controllers/iofog-controller.js | 16 -- src/data/managers/hw-info-manager.js | 12 -- src/data/managers/usb-info-manager.js | 12 -- src/data/models/architecture.js | 16 -- src/data/models/hwinfo.js | 33 ---- src/data/models/usbinfo.js | 33 ---- src/helpers/system-naming.js | 2 +- src/routes/iofog.js | 61 ------- src/schemas/fog-platform-spec.js | 6 +- src/schemas/iofog.js | 18 +- src/services/catalog-service.js | 20 --- src/services/fog-platform-service.js | 20 --- src/services/iofog-service.js | 155 +----------------- test/src/controllers/iofog-controller.test.js | 94 ----------- test/src/routes/iofog-routes.test.js | 11 ++ test/src/services/catalog-service.test.js | 20 --- .../src/services/fog-platform-service.test.js | 6 +- test/src/services/iofog-service.test.js | 53 +----- 18 files changed, 18 insertions(+), 570 deletions(-) delete mode 100644 src/data/managers/hw-info-manager.js delete mode 100644 src/data/managers/usb-info-manager.js delete mode 100644 src/data/models/hwinfo.js delete mode 100644 src/data/models/usbinfo.js create mode 100644 test/src/routes/iofog-routes.test.js diff --git a/src/controllers/iofog-controller.js b/src/controllers/iofog-controller.js index 61d72345..a49ea8be 100644 --- a/src/controllers/iofog-controller.js +++ b/src/controllers/iofog-controller.js @@ -67,20 +67,6 @@ async function setFogRebootCommandEndPoint (req) { return FogService.setFogRebootCommandEndPoint(fog, false) } -async function getHalHardwareInfoEndPoint (req) { - const uuidObj = { - uuid: req.params.uuid - } - return FogService.getHalHardwareInfoEndPoint(uuidObj, false) -} - -async function getHalUsbInfoEndPoint (req) { - const uuidObj = { - uuid: req.params.uuid - } - return FogService.getHalUsbInfoEndPoint(uuidObj, false) -} - async function setFogPruneCommandEndPoint (req) { const fog = { uuid: req.params.uuid @@ -116,8 +102,6 @@ module.exports = { generateProvisioningKeyEndPoint: (generateProvisionKeyEndPoint), setFogVersionCommandEndPoint, setFogRebootCommandEndPoint, - getHalHardwareInfoEndPoint, - getHalUsbInfoEndPoint, setFogPruneCommandEndPoint, enableNodeExecEndPoint, disableNodeExecEndPoint diff --git a/src/data/managers/hw-info-manager.js b/src/data/managers/hw-info-manager.js deleted file mode 100644 index c65e89a3..00000000 --- a/src/data/managers/hw-info-manager.js +++ /dev/null @@ -1,12 +0,0 @@ -const BaseManager = require('./base-manager') -const models = require('../models') -const HWInfo = models.HWInfo - -class HWInfoManager extends BaseManager { - getEntity () { - return HWInfo - } -} - -const instance = new HWInfoManager() -module.exports = instance diff --git a/src/data/managers/usb-info-manager.js b/src/data/managers/usb-info-manager.js deleted file mode 100644 index a3e81871..00000000 --- a/src/data/managers/usb-info-manager.js +++ /dev/null @@ -1,12 +0,0 @@ -const BaseManager = require('./base-manager') -const models = require('../models') -const USBInfo = models.USBInfo - -class USBInfoManager extends BaseManager { - getEntity () { - return USBInfo - } -} - -const instance = new USBInfoManager() -module.exports = instance diff --git a/src/data/models/architecture.js b/src/data/models/architecture.js index 8ec11768..b3a53a56 100644 --- a/src/data/models/architecture.js +++ b/src/data/models/architecture.js @@ -33,22 +33,6 @@ module.exports = (sequelize, DataTypes) => { }, as: 'networkCatalogItem' }) - - Architecture.belongsTo(models.CatalogItem, { - foreignKey: { - name: 'halCatalogItemId', - field: 'hal_catalog_item_id' - }, - as: 'halCatalogItem' - }) - - Architecture.belongsTo(models.CatalogItem, { - foreignKey: { - name: 'bluetoothCatalogItemId', - field: 'bluetooth_catalog_item_id' - }, - as: 'bluetoothCatalogItem' - }) } return Architecture } diff --git a/src/data/models/hwinfo.js b/src/data/models/hwinfo.js deleted file mode 100644 index 10e73a1e..00000000 --- a/src/data/models/hwinfo.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' -module.exports = (sequelize, DataTypes) => { - const HWInfo = sequelize.define('HWInfo', { - id: { - type: DataTypes.INTEGER, - primaryKey: true, - autoIncrement: true, - allowNull: false, - field: 'id' - }, - info: { - type: DataTypes.TEXT, - defaultValue: ' ', - field: 'info' - } - }, { - // add the timestamp attributes (updatedAt, createdAt) - tableName: 'HWInfos', - timestamps: true, - underscored: true - }) - HWInfo.associate = function (models) { - HWInfo.belongsTo(models.Fog, { - foreignKey: { - name: 'iofogUuid', - field: 'iofog_uuid' - }, - as: 'iofog', - onDelete: 'cascade' - }) - } - return HWInfo -} diff --git a/src/data/models/usbinfo.js b/src/data/models/usbinfo.js deleted file mode 100644 index 6458bc90..00000000 --- a/src/data/models/usbinfo.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' -module.exports = (sequelize, DataTypes) => { - const USBInfo = sequelize.define('USBInfo', { - id: { - type: DataTypes.INTEGER, - primaryKey: true, - autoIncrement: true, - allowNull: false, - field: 'id' - }, - info: { - type: DataTypes.TEXT, - defaultValue: ' ', - field: 'info' - } - }, { - tableName: 'USBInfos', - // add the timestamp attributes (updatedAt, createdAt) - timestamps: true, - underscored: true - }) - USBInfo.associate = function (models) { - USBInfo.belongsTo(models.Fog, { - foreignKey: { - name: 'iofogUuid', - field: 'iofog_uuid' - }, - as: 'iofog', - onDelete: 'cascade' - }) - } - return USBInfo -} diff --git a/src/helpers/system-naming.js b/src/helpers/system-naming.js index 3570ff60..c7a92b1d 100644 --- a/src/helpers/system-naming.js +++ b/src/helpers/system-naming.js @@ -3,7 +3,7 @@ const Errors = require('./errors') const ApplicationManager = require('../data/managers/application-manager') const MicroserviceManager = require('../data/managers/microservice-manager') -const SYSTEM_MICROSERVICE_TYPES = ['router', 'hal', 'ble', 'debug', 'nats'] +const SYSTEM_MICROSERVICE_TYPES = ['router', 'debug', 'nats'] const getSystemAppName = (fogName) => `system-${fogName}` diff --git a/src/routes/iofog.js b/src/routes/iofog.js index dce4c412..3dd4e9d1 100644 --- a/src/routes/iofog.js +++ b/src/routes/iofog.js @@ -302,67 +302,6 @@ module.exports = [ }) } }, - { - method: 'get', - path: '/api/v3/iofog/:uuid/hal/hw', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_SUCCESS - const errCodes = [ - { - code: 401, - errors: [Errors.AuthenticationError] - }, - { - code: 404, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const getHalHardwareInfo = ResponseDecorator.handleErrors(FogController.getHalHardwareInfoEndPoint, - successCode, errCodes) - const responseObject = await getHalHardwareInfo(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, - { - method: 'get', - path: '/api/v3/iofog/:uuid/hal/usb', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_SUCCESS - const errCodes = [ - { - code: 401, - errors: [Errors.AuthenticationError] - }, - { - code: 404, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const getHalUsbInfo = ResponseDecorator.handleErrors(FogController.getHalUsbInfoEndPoint, successCode, errCodes) - const responseObject = await getHalUsbInfo(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, { method: 'post', path: '/api/v3/iofog/:uuid/prune', diff --git a/src/schemas/fog-platform-spec.js b/src/schemas/fog-platform-spec.js index 84c09c39..6d3309c0 100644 --- a/src/schemas/fog-platform-spec.js +++ b/src/schemas/fog-platform-spec.js @@ -54,8 +54,6 @@ const fogPlatformSpec = { jsMemoryStoreSize: { type: 'string', maxLength: 32 }, networkRouter: { type: ['string', 'null'] }, containerEngine: { type: 'string', enum: ['edgelet', 'docker', 'podman'] }, - bluetoothEnabled: { type: 'boolean' }, - abstractedHardwareEnabled: { type: 'boolean' }, tags: { type: 'array', items: { @@ -159,9 +157,7 @@ const FOG_PLATFORM_SPEC_SCALAR_FIELDS = [ 'jsStorageSize', 'jsMemoryStoreSize', 'networkRouter', - 'containerEngine', - 'bluetoothEnabled', - 'abstractedHardwareEnabled' + 'containerEngine' ] const FOG_PLATFORM_SPEC_ARRAY_FIELDS = [ diff --git a/src/schemas/iofog.js b/src/schemas/iofog.js index 3a3bc130..c4dab556 100644 --- a/src/schemas/iofog.js +++ b/src/schemas/iofog.js @@ -22,10 +22,7 @@ const iofogCreate = { logFileCount: { type: 'integer', minimum: 0 }, statusFrequency: { type: 'integer', minimum: 0 }, changeFrequency: { type: 'integer', minimum: 0 }, - deviceScanFrequency: { type: 'integer', minimum: 0 }, - bluetoothEnabled: { type: 'boolean' }, watchdogEnabled: { type: 'boolean' }, - abstractedHardwareEnabled: { type: 'boolean' }, archId: { type: 'integer', minimum: 0, maximum: 4 }, pruningFrequency: { type: 'integer', minimum: 0 }, availableDiskThreshold: { type: 'integer', minimum: 0 }, @@ -99,10 +96,7 @@ const iofogUpdate = { logFileCount: { type: 'integer', minimum: 0 }, statusFrequency: { type: 'integer', minimum: 0 }, changeFrequency: { type: 'integer', minimum: 0 }, - deviceScanFrequency: { type: 'integer', minimum: 0 }, - bluetoothEnabled: { type: 'boolean' }, watchdogEnabled: { type: 'boolean' }, - abstractedHardwareEnabled: { type: 'boolean' }, archId: { type: 'integer', minimum: 0, maximum: 4 }, pruningFrequency: { type: 'integer', minimum: 0 }, availableDiskThreshold: { type: 'integer', minimum: 0 }, @@ -232,16 +226,6 @@ const filter = { additionalProperties: true } -const halGet = { - id: '/halGet', - type: 'object', - properties: { - uuid: { type: 'string' } - }, - required: ['uuid'], - additionalProperties: true -} - const iofogPrune = { id: '/iofogPrune', type: 'object', @@ -294,6 +278,6 @@ const disableNodeExec = { module.exports = { mainSchemas: [iofogCreate, iofogUpdate, iofogDelete, iofogGet, iofogGenerateProvision, iofogSetVersionCommand, - iofogReboot, iofogFilters, halGet, iofogPrune, defaultRouterCreate, iofogTag, enableNodeExec, disableNodeExec], + iofogReboot, iofogFilters, iofogPrune, defaultRouterCreate, iofogTag, enableNodeExec, disableNodeExec], innerSchemas: [filter, iofogTag] } diff --git a/src/services/catalog-service.js b/src/services/catalog-service.js index 603df1fc..59ce930e 100644 --- a/src/services/catalog-service.js +++ b/src/services/catalog-service.js @@ -157,24 +157,6 @@ async function getDebugCatalogItem (transaction) { }, transaction) } -async function getBluetoothCatalogItem (transaction) { - return CatalogItemManager.findOne({ - name: 'RESTBlue', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) -} - -async function getHalCatalogItem (transaction) { - return CatalogItemManager.findOne({ - name: 'HAL', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) -} - const _checkForDuplicateName = async function (name, item, transaction) { if (name) { const where = (item && item.id) @@ -371,8 +353,6 @@ module.exports = { getCatalogItem, getSystemCatalogItem, getNatsCatalogItem, - getBluetoothCatalogItem, - getHalCatalogItem, getRouterCatalogItem, getDebugCatalogItem } diff --git a/src/services/fog-platform-service.js b/src/services/fog-platform-service.js index 9b9c2049..f2f73b1c 100644 --- a/src/services/fog-platform-service.js +++ b/src/services/fog-platform-service.js @@ -45,10 +45,6 @@ function buildFogDataFromSpecAndFog (fog, spec) { jsMemoryStoreSize: spec.jsMemoryStoreSize, networkRouter: spec.networkRouter, containerEngine: spec.containerEngine || fog.containerEngine, - bluetoothEnabled: spec.bluetoothEnabled != null ? spec.bluetoothEnabled : fog.bluetoothEnabled, - abstractedHardwareEnabled: spec.abstractedHardwareEnabled != null - ? spec.abstractedHardwareEnabled - : fog.abstractedHardwareEnabled, tags: fogTags.length > 0 ? fogTags : specTags } } @@ -389,22 +385,6 @@ async function reconcileFogPlatform (fogUuid, prep, transaction) { await IofogService._updateMicroserviceExtraHosts(fogUuid, spec.host, transaction) } - if (fog.abstractedHardwareEnabled === true && spec.abstractedHardwareEnabled === false) { - await IofogService._deleteHalMicroserviceByFog(fogData, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } else if (fog.abstractedHardwareEnabled === false && spec.abstractedHardwareEnabled === true) { - await IofogService._createHalMicroserviceForFog(fogData, fog, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } - - if (fog.bluetoothEnabled === true && spec.bluetoothEnabled === false) { - await IofogService._deleteBluetoothMicroserviceByFog(fogData, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } else if (fog.bluetoothEnabled === false && spec.bluetoothEnabled === true) { - await IofogService._createBluetoothMicroserviceForFog(fogData, fog, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } - if (prep.isFirstReconcile) { await ChangeTrackingService.create(fogUuid, transaction) } diff --git a/src/services/iofog-service.js b/src/services/iofog-service.js index affde2fa..c602d6a1 100644 --- a/src/services/iofog-service.js +++ b/src/services/iofog-service.js @@ -10,8 +10,6 @@ const NatsService = require('./nats-service') const Errors = require('../helpers/errors') const ErrorMessages = require('../helpers/error-messages') const Validator = require('../schemas') -const HWInfoManager = require('../data/managers/hw-info-manager') -const USBInfoManager = require('../data/managers/usb-info-manager') const CatalogService = require('./catalog-service') const MicroserviceManager = require('../data/managers/microservice-manager') const ApplicationManager = require('../data/managers/application-manager') @@ -355,10 +353,7 @@ async function createFogEndPoint (fogData, isCLI, transaction) { logFileCount: fogData.logFileCount, statusFrequency: fogData.statusFrequency, changeFrequency: fogData.changeFrequency, - deviceScanFrequency: fogData.deviceScanFrequency, - bluetoothEnabled: fogData.bluetoothEnabled, watchdogEnabled: fogData.watchdogEnabled, - abstractedHardwareEnabled: fogData.abstractedHardwareEnabled, archId: _resolveArchId(fogData), logLevel: fogData.logLevel, edgeGuardFrequency: fogData.edgeGuardFrequency, @@ -370,7 +365,7 @@ async function createFogEndPoint (fogData, isCLI, transaction) { timeZone: fogData.timeZone } - if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off')) { + if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off' && fogData.gpsMode !== 'auto')) { createFogData.gpsMode = 'manual' } else if (fogData.gpsMode === 'dynamic' && fogData.gpsDevice) { createFogData.gpsMode = fogData.gpsMode @@ -484,11 +479,8 @@ async function updateFogEndPoint (fogData, isCLI, transaction) { logFileCount: fogData.logFileCount, statusFrequency: fogData.statusFrequency, changeFrequency: fogData.changeFrequency, - deviceScanFrequency: fogData.deviceScanFrequency, - bluetoothEnabled: fogData.bluetoothEnabled, watchdogEnabled: fogData.watchdogEnabled, isSystem: fogData.isSystem, - abstractedHardwareEnabled: fogData.abstractedHardwareEnabled, archId: _resolveArchId(fogData), logLevel: fogData.logLevel, pruningFrequency: fogData.pruningFrequency, @@ -498,7 +490,7 @@ async function updateFogEndPoint (fogData, isCLI, transaction) { timeZone: fogData.timeZone } - if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off')) { + if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off' && fogData.gpsMode !== 'auto')) { updateFogData.gpsMode = 'manual' } else if (fogData.gpsMode === 'dynamic' && fogData.gpsDevice) { updateFogData.gpsMode = fogData.gpsMode @@ -1014,36 +1006,6 @@ async function setFogRebootCommandEndPoint (fogData, isCLI, transaction) { await ChangeTrackingService.update(fogData.uuid, ChangeTrackingService.events.reboot, transaction) } -async function getHalHardwareInfoEndPoint (uuidObj, isCLI, transaction) { - await Validator.validate(uuidObj, Validator.schemas.halGet) - - const fog = await FogManager.findOne({ - uuid: uuidObj.uuid - }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, uuidObj.uuid)) - } - - return HWInfoManager.findOne({ - iofogUuid: uuidObj.uuid - }, transaction) -} - -async function getHalUsbInfoEndPoint (uuidObj, isCLI, transaction) { - await Validator.validate(uuidObj, Validator.schemas.halGet) - - const fog = await FogManager.findOne({ - uuid: uuidObj.uuid - }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, uuidObj.uuid)) - } - - return USBInfoManager.findOne({ - iofogUuid: uuidObj.uuid - }, transaction) -} - function _filterFogs (fogs, filters) { if (!filters) { return fogs @@ -1107,60 +1069,6 @@ async function _processDeleteCommand (fog, transaction) { await FogManager.delete({ uuid: fog.uuid }, transaction) } -async function _createHalMicroserviceForFog (fogData, oldFog, transaction) { - const halItem = await CatalogService.getHalCatalogItem(transaction) - const systemMicroserviceName = getSystemMicroserviceName('hal') - const fogForName = (fogData && fogData.name) ? fogData : oldFog - - const halMicroserviceData = { - uuid: AppHelper.generateUUID(), - name: systemMicroserviceName, - config: '{}', - catalogItemId: halItem.id, - iofogUuid: fogData.uuid, - hostNetworkMode: true, - isPrivileged: true, - logSize: Constants.MICROSERVICE_DEFAULT_LOG_SIZE, - schedule: 1, - configLastUpdated: Date.now() - } - - const application = await ensureSystemApplication(fogForName, transaction) - halMicroserviceData.applicationId = application.id - const existingMicroservice = await MicroserviceManager.findOne({ - name: systemMicroserviceName, - applicationId: application.id - }, transaction) - if (!existingMicroservice) { - await MicroserviceManager.create(halMicroserviceData, transaction) - await MicroserviceStatusManager.create({ microserviceUuid: halMicroserviceData.uuid }, transaction) - await MicroserviceExecStatusManager.create({ microserviceUuid: halMicroserviceData.uuid }, transaction) - } -} - -async function _deleteHalMicroserviceByFog (fogData, transaction) { - const halItem = await CatalogService.getHalCatalogItem(transaction) - const deleteHalMicroserviceData = { - iofogUuid: fogData.uuid, - catalogItemId: halItem.id - } - - const fog = await FogManager.findOne({ uuid: fogData.uuid }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, fogData.uuid)) - } - const systemAppName = getSystemAppName(fog.name) - const legacySystemAppName = getLegacySystemAppName(fog.uuid) - let application = await ApplicationManager.findOne({ name: systemAppName }, transaction) - if (!application) { - application = await ApplicationManager.findOne({ name: legacySystemAppName }, transaction) - } - if (application) { - deleteHalMicroserviceData.applicationId = application.id - await MicroserviceManager.delete(deleteHalMicroserviceData, transaction) - } -} - async function _deleteNatsMicroserviceByFog (fogData, transaction) { const natsItem = await CatalogService.getNatsCatalogItem(transaction) if (!natsItem) return @@ -1185,59 +1093,6 @@ async function _deleteNatsMicroserviceByFog (fogData, transaction) { } } -async function _createBluetoothMicroserviceForFog (fogData, oldFog, transaction) { - const bluetoothItem = await CatalogService.getBluetoothCatalogItem(transaction) - const systemMicroserviceName = getSystemMicroserviceName('ble') - const fogForName = (fogData && fogData.name) ? fogData : oldFog - - const bluetoothMicroserviceData = { - uuid: AppHelper.generateUUID(), - name: systemMicroserviceName, - config: '{}', - catalogItemId: bluetoothItem.id, - iofogUuid: fogData.uuid, - hostNetworkMode: true, - isPrivileged: true, - logSize: Constants.MICROSERVICE_DEFAULT_LOG_SIZE, - schedule: 1, - configLastUpdated: Date.now() - } - - const application = await ensureSystemApplication(fogForName, transaction) - bluetoothMicroserviceData.applicationId = application.id - const existingMicroservice = await MicroserviceManager.findOne({ - name: systemMicroserviceName, - applicationId: application.id - }, transaction) - if (!existingMicroservice) { - await MicroserviceManager.create(bluetoothMicroserviceData, transaction) - await MicroserviceStatusManager.create({ microserviceUuid: bluetoothMicroserviceData.uuid }, transaction) - await MicroserviceExecStatusManager.create({ microserviceUuid: bluetoothMicroserviceData.uuid }, transaction) - } -} - -async function _deleteBluetoothMicroserviceByFog (fogData, transaction) { - const bluetoothItem = await CatalogService.getBluetoothCatalogItem(transaction) - const deleteBluetoothMicroserviceData = { - iofogUuid: fogData.uuid, - catalogItemId: bluetoothItem.id - } - const fog = await FogManager.findOne({ uuid: fogData.uuid }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, fogData.uuid)) - } - const systemAppName = getSystemAppName(fog.name) - const legacySystemAppName = getLegacySystemAppName(fog.uuid) - let application = await ApplicationManager.findOne({ name: systemAppName }, transaction) - if (!application) { - application = await ApplicationManager.findOne({ name: legacySystemAppName }, transaction) - } - if (application) { - deleteBluetoothMicroserviceData.applicationId = application.id - await MicroserviceManager.delete(deleteBluetoothMicroserviceData, transaction) - } -} - async function setFogPruneCommandEndPoint (fogData, isCLI, transaction) { await Validator.validate(fogData, Validator.schemas.iofogPrune) @@ -1545,8 +1400,6 @@ module.exports = { generateProvisioningKeyEndPoint: TransactionDecorator.generateTransaction(generateProvisioningKeyEndPoint), setFogVersionCommandEndPoint: TransactionDecorator.generateTransaction(setFogVersionCommandEndPoint), setFogRebootCommandEndPoint: TransactionDecorator.generateTransaction(setFogRebootCommandEndPoint), - getHalHardwareInfoEndPoint: TransactionDecorator.generateTransaction(getHalHardwareInfoEndPoint), - getHalUsbInfoEndPoint: TransactionDecorator.generateTransaction(getHalUsbInfoEndPoint), getFog, refreshProvisionKeyForFog, setFogPruneCommandEndPoint: TransactionDecorator.generateTransaction(setFogPruneCommandEndPoint), @@ -1565,9 +1418,5 @@ module.exports = { _processDeleteCommand, _reconcileNatsCertificatesOnHostChange, _deleteNatsMicroserviceByFog, - _createHalMicroserviceForFog, - _deleteHalMicroserviceByFog, - _createBluetoothMicroserviceForFog, - _deleteBluetoothMicroserviceByFog, _updateMicroserviceExtraHosts } diff --git a/test/src/controllers/iofog-controller.test.js b/test/src/controllers/iofog-controller.test.js index 18721235..691a1421 100644 --- a/test/src/controllers/iofog-controller.test.js +++ b/test/src/controllers/iofog-controller.test.js @@ -30,10 +30,7 @@ describe('ioFog Controller', () => { def('logFileCount', () => 8) def('statusFrequency', 6) def('changeFrequency', 18) - def('deviceScanFrequency', 28) - def('bluetoothEnabled', () => false) def('watchdogEnabled', () => true) - def('abstractedHardwareEnabled', () => false) def('archId', () => 0) def('req', () => ({ @@ -53,10 +50,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, })) @@ -86,10 +80,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, false) }) @@ -130,10 +121,7 @@ describe('ioFog Controller', () => { def('logFileCount', () => 8) def('statusFrequency', 6) def('changeFrequency', 18) - def('deviceScanFrequency', 28) - def('bluetoothEnabled', () => false) def('watchdogEnabled', () => true) - def('abstractedHardwareEnabled', () => false) def('archId', () => 0) def('req', () => ({ @@ -156,10 +144,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, })) @@ -189,10 +174,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, false) }) @@ -490,80 +472,4 @@ describe('ioFog Controller', () => { }) }) }) - - describe('getHalHardwareInfoEndPoint()', () => { - def('user', () => 'user!') - def('uuid', () => 'testUuid') - - def('req', () => ({ - params: { - uuid: $uuid, - }, - })) - def('response', () => Promise.resolve()) - def('subject', () => $subject.getHalHardwareInfoEndPoint($req)) - - beforeEach(() => { - $sandbox.stub(ioFogService, 'getHalHardwareInfoEndPoint').returns($response) - }) - - it('calls ioFogService.getHalHardwareInfoEndPoint with correct args', async () => { - await $subject - expect(ioFogService.getHalHardwareInfoEndPoint).to.have.been.calledWith({ uuid: $uuid }, false) - }) - - context('when ioFogService#getHalHardwareInfoEndPoint fails', () => { - const error = 'Error!' - - def('response', () => Promise.reject(error)) - - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when ioFogService#getHalHardwareInfoEndPoint succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.equal(undefined) - }) - }) - }) - - describe('getHalUsbInfoEndPoint.()', () => { - def('user', () => 'user!') - def('uuid', () => 'testUuid') - - def('req', () => ({ - params: { - uuid: $uuid, - }, - })) - def('response', () => Promise.resolve({ info: undefined })) - def('subject', () => $subject.getHalUsbInfoEndPoint($req)) - - beforeEach(() => { - $sandbox.stub(ioFogService, 'getHalUsbInfoEndPoint').returns($response) - }) - - it('calls ioFogService.getHalUsbInfoEndPoint with correct args', async () => { - await $subject - expect(ioFogService.getHalUsbInfoEndPoint).to.have.been.calledWith({ uuid: $uuid }, false) - }) - - context('when ioFogService#getHalUsbInfoEndPoint fails', () => { - const error = 'Error!' - - def('response', () => Promise.reject(error)) - - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when ioFogService#getHalUsbInfoEndPoint succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.have.property('info') - }) - }) - }) }) diff --git a/test/src/routes/iofog-routes.test.js b/test/src/routes/iofog-routes.test.js new file mode 100644 index 00000000..1979dd63 --- /dev/null +++ b/test/src/routes/iofog-routes.test.js @@ -0,0 +1,11 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('ioFog routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/iofog.js'), 'utf8') + + it('does not register hardware inventory user routes', () => { + expect(source).to.not.include('/hal/') + }) +}) diff --git a/test/src/services/catalog-service.test.js b/test/src/services/catalog-service.test.js index 86efad67..700106ae 100644 --- a/test/src/services/catalog-service.test.js +++ b/test/src/services/catalog-service.test.js @@ -381,25 +381,5 @@ describe('Catalog Service', () => { registry_id: 1 }, transaction) }) - - it('.getBluetoothCatalogItem() queries RESTBlue system item', async () => { - await CatalogService.getBluetoothCatalogItem(transaction) - expect(CatalogItemManager.findOne).to.have.been.calledWith({ - name: 'RESTBlue', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) - }) - - it('.getHalCatalogItem() queries HAL system item', async () => { - await CatalogService.getHalCatalogItem(transaction) - expect(CatalogItemManager.findOne).to.have.been.calledWith({ - name: 'HAL', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) - }) }) }) diff --git a/test/src/services/fog-platform-service.test.js b/test/src/services/fog-platform-service.test.js index 87385414..698e423a 100644 --- a/test/src/services/fog-platform-service.test.js +++ b/test/src/services/fog-platform-service.test.js @@ -61,8 +61,6 @@ describe('Fog platform service', () => { name: 'edge-a', isSystem: false, host: '10.0.0.5', - bluetoothEnabled: false, - abstractedHardwareEnabled: false, containerEngine: 'edgelet', tags: [] } @@ -71,9 +69,7 @@ describe('Fog platform service', () => { natsMode: 'leaf', host: '10.0.0.5', messagingPort: 5671, - containerEngine: 'edgelet', - bluetoothEnabled: false, - abstractedHardwareEnabled: false + containerEngine: 'edgelet' } const parsedSpec = { fogUuid, generation: 2, spec } const router = { diff --git a/test/src/services/iofog-service.test.js b/test/src/services/iofog-service.test.js index ff4660a4..0fac5d3e 100644 --- a/test/src/services/iofog-service.test.js +++ b/test/src/services/iofog-service.test.js @@ -12,7 +12,6 @@ const NatsConnectionManager = require('../../../src/data/managers/nats-connectio const AppHelper = require('../../../src/helpers/app-helper') const Validator = require('../../../src/schemas') const ChangeTrackingService = require('../../../src/services/change-tracking-service') -const CatalogService = require('../../../src/services/catalog-service') const MicroserviceManager = require('../../../src/data/managers/microservice-manager') const MicroserviceService = require('../../../src/services/microservices-service') const ApplicationManager = require('../../../src/data/managers/application-manager') @@ -21,8 +20,6 @@ const FogPublicKeyManager = require('../../../src/data/managers/iofog-public-key const TagsManager = require('../../../src/data/managers/tags-manager') const ioFogProvisionKeyManager = require('../../../src/data/managers/iofog-provision-key-manager') const ioFogVersionCommandManager = require('../../../src/data/managers/iofog-version-command-manager') -const HWInfoManager = require('../../../src/data/managers/hw-info-manager') -const USBInfoManager = require('../../../src/data/managers/usb-info-manager') const Errors = require('../../../src/helpers/errors') const config = require('../../../src/config') const FogPlatformSpecManager = require('../../../src/data/managers/fog-platform-spec-manager') @@ -137,9 +134,7 @@ describe('ioFog Service', () => { pruningFrequency: 10, availableDiskThreshold: 20, logLevel: 'INFO', - routerMode: 'edge', - abstractedHardwareEnabled: false, - bluetoothEnabled: false + routerMode: 'edge' } def('subject', () => $subject.createFogEndPoint(fogData, isCLI, transaction)) @@ -187,14 +182,6 @@ describe('ioFog Service', () => { expect(NatsService.ensureNatsForFog).to.not.have.been.called }) - it('does not run HAL/Bluetooth catalog work on the synchronous path', async () => { - $sandbox.stub(CatalogService, 'getHalCatalogItem').resolves({ id: 1 }) - $sandbox.stub(CatalogService, 'getBluetoothCatalogItem').resolves({ id: 2 }) - await $subject - expect(CatalogService.getHalCatalogItem).to.not.have.been.called - expect(CatalogService.getBluetoothCatalogItem).to.not.have.been.called - }) - context('when validation fails', () => { const validationError = new Error('validation failed') @@ -871,42 +858,4 @@ describe('ioFog Service', () => { expect(ChangeTrackingService.update).to.have.been.calledWith(uuid, ChangeTrackingService.events.reboot, transaction) }) }) - - describe('.getHalHardwareInfoEndPoint()', () => { - const uuidObj = { uuid: 'testUuid' } - const hwInfo = { cpu: 'arm64' } - - def('subject', () => $subject.getHalHardwareInfoEndPoint(uuidObj, isCLI, transaction)) - - beforeEach(() => { - $sandbox.stub(Validator, 'validate').resolves(true) - $sandbox.stub(ioFogManager, 'findOne').resolves({ uuid: uuidObj.uuid }) - $sandbox.stub(HWInfoManager, 'findOne').resolves(hwInfo) - }) - - it('returns HAL hardware info', async () => { - const result = await $subject - expect(result).to.equal(hwInfo) - expect(HWInfoManager.findOne).to.have.been.calledWith({ iofogUuid: uuidObj.uuid }, transaction) - }) - }) - - describe('.getHalUsbInfoEndPoint()', () => { - const uuidObj = { uuid: 'testUuid' } - const usbInfo = { devices: [] } - - def('subject', () => $subject.getHalUsbInfoEndPoint(uuidObj, isCLI, transaction)) - - beforeEach(() => { - $sandbox.stub(Validator, 'validate').resolves(true) - $sandbox.stub(ioFogManager, 'findOne').resolves({ uuid: uuidObj.uuid }) - $sandbox.stub(USBInfoManager, 'findOne').resolves(usbInfo) - }) - - it('returns HAL USB info', async () => { - const result = await $subject - expect(result).to.equal(usbInfo) - expect(USBInfoManager.findOne).to.have.been.calledWith({ iofogUuid: uuidObj.uuid }, transaction) - }) - }) }) From bef7762ec9c510c2c5864c97d93dd99838bffa36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:51:13 +0300 Subject: [PATCH 10/55] Catalog models, RuntimeClasses, and microservice templates in RBAC and swagger. Viewer stays read-only; SRE, developer, and admin get full verbs including link. --- docs/rbac-reference.md | 11 +- docs/swagger.yaml | 2193 +++++++++++++++++++++++----- src/config/rbac-resources.yaml | 102 +- src/config/rbac-system-roles.js | 6 +- test/src/config/rbac-audit.test.js | 44 + 5 files changed, 1943 insertions(+), 413 deletions(-) create mode 100644 test/src/config/rbac-audit.test.js diff --git a/docs/rbac-reference.md b/docs/rbac-reference.md index 951aafb2..3e6bec2a 100644 --- a/docs/rbac-reference.md +++ b/docs/rbac-reference.md @@ -1,7 +1,7 @@ # Controller RBAC — operator reference **Audience:** Platform operators, SREs, and integrators configuring access to Controller v3.8 -**Applies to:** User and admin HTTP APIs under `/api/v3/*` (not Edgelet agent wire protocol) +**Applies to:** User and admin HTTP APIs under `/api/v3/*` (not Edgelet agent wire protocol). Includes fleet models, RuntimeClass, and microservice templates. ## Overview @@ -79,13 +79,14 @@ The table below lists **user API** resources in `rbac-resources.yaml` and which | applications | ✓ | ✓ | ✓ | Replaces legacy `flows` | | systemApplications | ✓ | read | ✓ | | | applicationTemplates | ✓ | ✓ | ✓ | | +| microserviceTemplates | ✓ | ✓ | ✓ | | | services | ✓ | ✓ | ✓ | | | router | ✓ | read | ✓ | | | cluster | ✓ | read | ✓ | v3.8 HA controllers | | natsOperator, natsBootstrap, natsHub | read / ✓ | read | read | SRE has full NATS CRUD except operator objects (read) | | natsAccounts, natsUsers, natsAccountRules, natsUserRules | ✓ | ✓ | ✓ | | | catalog, registries | ✓ | ✓ | ✓ | | -| secrets, configMaps, volumeMounts | ✓ | ✓ | ✓ | | +| secrets, configMaps, volumeMounts, models, runtimeClasses | ✓ | ✓ | ✓ | Model/RuntimeClass link uses `patch` (same as volumeMounts) | | tunnels | ✓ | read | — | Viewer intentionally omits exec/tunnel paths | | certificates, capabilities | ✓ | ✓ | ✓ | | | execSessions, logs | ✓ | ✓ | — | | @@ -139,6 +140,10 @@ Some sub-resource actions map to `patch` (for example microservice start/stop). | `POST /api/v3/agent/controller/register` | Added under **`agent`** resource (fog token) | | OIDC auth routes (OAuth BFF, interactions) | Catalogued under **`users`** with auth-only verbs | +Fleet models, RuntimeClass, and microservice templates are user RBAC resources (`models`, `runtimeClasses`, `microserviceTemplates`). Viewer gets `get`/`list`. SRE, developer, and admin get full CRUD including link (`patch` on `…/:name/link`). Agent `GET /api/v3/agent/models` and `GET /api/v3/agent/runtimeClasses` are fog-token routes catalogued under **`agent`** for drift auditing only. + +Removed user HAL/USB inventory routes (`GET /api/v3/iofog/:uuid/hal/hw`, `GET /api/v3/iofog/:uuid/hal/usb`) and agent HAL PUT routes have no yaml entries. + Orphan RBAC entries for removed APIs must not reappear. CI and local checks enforce this (see Maintenance). ## Custom roles @@ -157,7 +162,7 @@ Keep `rbac-resources.yaml`, live routes, and system roles aligned when adding or ```bash nvm use 24 -# Compare Express routes to rbac-resources.yaml (243 routes as) +# Compare Express routes to rbac-resources.yaml npm run rbac-audit # grep gates — banned legacy terms must be absent; diff --git a/docs/swagger.yaml b/docs/swagger.yaml index cc215ad9..586eadb6 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,6 +1,6 @@ openapi : "3.0.0" info: - version: 3.8.2 + version: 3.9.0 title: Cloud Native Controller for Edge Native Workloads paths: /live: @@ -400,52 +400,12 @@ paths: description: Not Authorized "500": description: Internal Server Error - "/iofog/{uuid}/hal/hw": - get: - tags: - - ioFog - summary: Retrieves HAL hardware info - operationId: getFogHalHardwareInfo - parameters: - - in: path - name: uuid - description: ioFog node id - required: true - schema: - type: string - security: - - authToken: [] - responses: - "200": - description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number - content: - application/json: - schema: - $ref: "#/components/schemas/HalInfo" - "401": - description: Not Authorized - "404": - description: Invalid Node Id - "500": - description: Internal Server Error - "/iofog/{uuid}/hal/usb": + /application: get: tags: - - ioFog - summary: Retrieves HAL USB info - operationId: getFogHalUsbInfo - parameters: - - in: path - name: uuid - description: ioFog node id - required: true - schema: - type: string + - Application + summary: Lists all applications + operationId: listApplication security: - authToken: [] responses: @@ -459,24 +419,29 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/HalInfo" - "401": - description: Not Authorized + $ref: "#/components/schemas/ApplicationListResponse" + "400": + description: Bad Request "404": - description: Invalid Node Id + description: Not Found "500": description: Internal Server Error - /application: - get: + post: tags: - Application - summary: Lists all applications - operationId: listApplication + summary: Creates an application + operationId: createApplication security: - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationCreateRequest" responses: - "200": - description: Success + "201": + description: Created headers: X-Timestamp: description: FogController server timestamp @@ -485,11 +450,11 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ApplicationListResponse" + $ref: "#/components/schemas/ApplicationCreateResponse" "400": description: Bad Request - "404": - description: Not Found + "401": + description: Not Authorized "500": description: Internal Server Error /application/system: @@ -615,6 +580,40 @@ paths: "500": description: Internal Server Error "/application/{name}": + put: + tags: + - Application + summary: Updates an application state (microservices, etc.) + operationId: updateApplication + parameters: + - in: path + name: name + description: Application name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationUpdateRequest" + responses: + "204": + description: Success + headers: + X-Timestamp: + description: FogController server timestamp + schema: + type: number + "400": + description: Bad Request + "404": + description: Not Found + "500": + description: Internal Server Error patch: tags: - Application @@ -633,14 +632,7 @@ paths: content: application/json: schema: - type: object - properties: - name: - type: string - description: - type: string - isActivated: - type: boolean + $ref: "#/components/schemas/ApplicationPatchRequest" required: true responses: "204": @@ -1439,88 +1431,58 @@ paths: description: Not Authorized "500": description: Internal Server Error - /agent/tunnel: + /agent/models: get: tags: - Agent - summary: Get an ioFog node tunnel configuration - operationId: getIOFogNodeTunnelConfig + summary: Gets fleet models linked to this fog + description: >- + Fog-token only. Returns models linked to the authenticated fog. + Rows missing uuid or name are omitted. A missing route is treated as + an empty list by the agent. + operationId: getAgentLinkedModels security: - authToken: [] responses: "200": description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number content: application/json: schema: - $ref: "#/components/schemas/IOFogNodeTunnelConfigResponse" + $ref: "#/components/schemas/AgentModelsListResponse" "401": description: Not Authorized - "404": - description: Tunnel Not Found "500": description: Internal Server Error - /agent/version: + /agent/runtimeClasses: get: tags: - Agent - summary: Get change version command - operationId: getChangeVersion + summary: Gets RuntimeClasses linked to this fog + description: >- + Fog-token only. Returns RuntimeClasses linked to the authenticated fog. + Rows missing name or handler are omitted. Docker and podman agents + store the snapshot and do not apply it. + operationId: getAgentLinkedRuntimeClasses security: - authToken: [] responses: "200": description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number content: application/json: schema: - $ref: "#/components/schemas/VersionCommandResponse" - "401": - description: Not Authorized - "404": - description: Version Command Not Found - "500": - description: Internal Server Error - /agent/hal/hw: - put: - tags: - - Agent - summary: Updates HAL hardware info - operationId: putHalHardwareInfo - security: - - authToken: [] - requestBody: - $ref: "#/components/requestBodies/HalInfo" - responses: - "204": - description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number - "400": - description: Bad Request + $ref: "#/components/schemas/AgentRuntimeClassesListResponse" "401": description: Not Authorized "500": description: Internal Server Error - /agent/hal/usb: + /agent/tunnel: get: tags: - - ioFog - summary: Retrieves HAL USB info - operationId: getAgentHalUsbInfo + - Agent + summary: Get an ioFog node tunnel configuration + operationId: getIOFogNodeTunnelConfig security: - authToken: [] responses: @@ -1534,34 +1496,37 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/HalInfo" + $ref: "#/components/schemas/IOFogNodeTunnelConfigResponse" "401": description: Not Authorized "404": - description: Invalid Node Id + description: Tunnel Not Found "500": description: Internal Server Error - put: + /agent/version: + get: tags: - Agent - summary: Updates HAL USB info - operationId: putHalUsbInfo + summary: Get change version command + operationId: getChangeVersion security: - authToken: [] - requestBody: - $ref: "#/components/requestBodies/HalInfo" responses: - "204": + "200": description: Success headers: X-Timestamp: description: FogController server timestamp schema: type: number - "400": - description: Bad Request + content: + application/json: + schema: + $ref: "#/components/schemas/VersionCommandResponse" "401": description: Not Authorized + "404": + description: Version Command Not Found "500": description: Internal Server Error /agent/delete-node: @@ -1791,6 +1756,12 @@ paths: tags: - Microservices summary: Creates a new microservice + description: >- + Optional template overlay: send template.name plus optional template.variables. + Request fields name, application, and iofogUuid/agentName win over the template. + Catalog models.items[].name are auto-linked to the target fog. Non-empty runtime + must exist as a RuntimeClass row and be in the fog availableRuntimes list. + registryId must resolve to type oci. operationId: createMicroservice security: - authToken: [] @@ -1823,6 +1794,11 @@ paths: tags: - Microservices summary: Creates a new microservice in an Application + description: >- + YAML kind Microservice. Optional spec.template applies the same overlay as + JSON POST /microservices (template.name plus optional template.variables). + Request fields metadata.name, spec.application, and spec.agent.name win over + the template. operationId: createMicroserviceYAML security: - authToken: [] @@ -2710,6 +2686,45 @@ paths: description: Not Found "500": description: Internal Server Error + "/microservices/{uuid}/models": + patch: + tags: + - Microservices + summary: Updates the microservice model catalog + description: >- + Catalog-only update. Sets the microserviceModels change flag only + (does not set microserviceList). Rebuilds the workload when the catalog + goes empty to non-empty or non-empty to empty, or when bindPath or + permissions change. Adding or removing items with the same bindPath + and permissions does not rebuild when the catalog is already non-empty. + Catalog items reference managed model names only. + operationId: patchMicroserviceModels + parameters: + - in: path + name: uuid + description: Microservice Uuid + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceCatalog" + responses: + "204": + description: Catalog updated + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error "/microservices/{uuid}/rebuild": patch: tags: @@ -5504,64 +5519,926 @@ paths: description: Not Found "500": description: Internal Server Error - /events: + + /models: get: tags: - - Events - summary: List audit events - description: Retrieve audit events with optional filters and pagination. - operationId: listEvents + - Models + summary: Returns fleet models + operationId: listModels security: - authToken: [] - parameters: - - in: query - name: limit - description: Maximum number of events to return (default 200, max 1000) - schema: - type: integer - minimum: 1 - maximum: 1000 - required: false - - in: query - name: offset - description: Number of events to skip before collecting results - schema: - type: integer - minimum: 0 - required: false - - in: query - name: startTime - description: Start of time range (Unix timestamp in ms or ISO 8601) - schema: - type: string - required: false - - in: query - name: endTime - description: End of time range (Unix timestamp in ms or ISO 8601) - schema: - type: string - required: false - - in: query - name: endpointType - description: Filter by endpoint type (agent or user) - schema: - type: string - required: false - - in: query - name: resourceType - description: Filter by resource type (agent, microservice, etc.) - schema: - type: string - required: false - - in: query - name: status - description: Filter by status (SUCCESS or FAILED) - schema: - type: string - required: false - - in: query - name: method - description: Filter by HTTP/WS method(s) + responses: + "200": + description: List of fleet models + content: + application/json: + schema: + $ref: "#/components/schemas/ModelsListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - Models + summary: Creates a fleet model + description: Server generates uuid. Name is unique and immutable after create. + operationId: createModel + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModelCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /models/yaml: + post: + tags: + - Models + summary: Creates a fleet model from YAML + operationId: createModelYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + model: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/models/{name}": + get: + tags: + - Models + summary: Gets a fleet model + description: Does not include linked fog uuids. Use GET /models/{name}/link. + operationId: getModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - Models + summary: Updates a fleet model + description: Name is immutable. + operationId: updateModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModelUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Models + summary: Deletes a fleet model + description: Refused when any microservice binds the model name. + operationId: deleteModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Model is bound by a microservice + "500": + description: Internal Server Error + + "/models/yaml/{name}": + put: + tags: + - Models + summary: Creates or updates a fleet model from YAML + operationId: upsertModelYaml + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + model: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + + "/models/{name}/link": + get: + tags: + - Models + summary: Gets fogs linked to a fleet model + operationId: getModelLink + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + post: + tags: + - Models + summary: Links a fleet model to fog nodes + operationId: linkModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Models + summary: Unlinks a fleet model from fog nodes + description: Refused when a microservice on that fog still binds the model name. + operationId: unlinkModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Model is bound by a microservice on the fog + "500": + description: Internal Server Error + + /runtimeClasses: + get: + tags: + - RuntimeClasses + summary: Returns fleet RuntimeClasses + operationId: listRuntimeClasses + security: + - authToken: [] + responses: + "200": + description: List of RuntimeClasses + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClassesListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - RuntimeClasses + summary: Creates a RuntimeClass + description: Name plus handler. Built-in catalog rows are not seeded. + operationId: createRuntimeClass + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClassCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /runtimeClasses/yaml: + post: + tags: + - RuntimeClasses + summary: Creates a RuntimeClass from YAML + operationId: createRuntimeClassYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + runtimeClass: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/runtimeClasses/{name}": + get: + tags: + - RuntimeClasses + summary: Gets a RuntimeClass + description: Does not include linked fog uuids. Use GET /runtimeClasses/{name}/link. + operationId: getRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - RuntimeClasses + summary: Updates a RuntimeClass + description: Name is immutable. Non-catalog handlers may bounce the data plane. + operationId: updateRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClassUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - RuntimeClasses + summary: Deletes a RuntimeClass + description: Refused when any microservice pins runtime to this name. + operationId: deleteRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: RuntimeClass is pinned by a microservice + "500": + description: Internal Server Error + + "/runtimeClasses/yaml/{name}": + put: + tags: + - RuntimeClasses + summary: Creates or updates a RuntimeClass from YAML + operationId: upsertRuntimeClassYaml + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + runtimeClass: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + + "/runtimeClasses/{name}/link": + get: + tags: + - RuntimeClasses + summary: Gets fogs linked to a RuntimeClass + operationId: getRuntimeClassLink + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + post: + tags: + - RuntimeClasses + summary: Links a RuntimeClass to fog nodes + description: Attach is refused when fog containerEngine is not edgelet. + operationId: linkRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Fog is not edgelet, or request invalid + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - RuntimeClasses + summary: Unlinks a RuntimeClass from fog nodes + description: Refused when a microservice on that fog still pins runtime to this name. + operationId: unlinkRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: RuntimeClass is pinned by a microservice on the fog + "500": + description: Internal Server Error + + /microserviceTemplates: + get: + tags: + - Microservice Templates + summary: Returns microservice templates + operationId: listMicroserviceTemplates + security: + - authToken: [] + responses: + "200": + description: List of microservice templates + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplatesListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - Microservice Templates + summary: Creates a microservice template + operationId: createMicroserviceTemplate + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplateCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /microserviceTemplates/yaml: + post: + tags: + - Microservice Templates + summary: Creates a microservice template from YAML + operationId: createMicroserviceTemplateYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + template: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/microserviceTemplates/{name}": + get: + tags: + - Microservice Templates + summary: Gets a microservice template + operationId: getMicroserviceTemplate + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - Microservice Templates + summary: Updates a microservice template + description: Name is immutable. + operationId: updateMicroserviceTemplate + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplateUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Microservice Templates + summary: Deletes a microservice template + operationId: deleteMicroserviceTemplate + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + + "/microserviceTemplates/yaml/{name}": + put: + tags: + - Microservice Templates + summary: Creates or updates a microservice template from YAML + operationId: upsertMicroserviceTemplateYaml + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + template: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + /events: + get: + tags: + - Events + summary: List audit events + description: Retrieve audit events with optional filters and pagination. + operationId: listEvents + security: + - authToken: [] + parameters: + - in: query + name: limit + description: Maximum number of events to return (default 200, max 1000) + schema: + type: integer + minimum: 1 + maximum: 1000 + required: false + - in: query + name: offset + description: Number of events to skip before collecting results + schema: + type: integer + minimum: 0 + required: false + - in: query + name: startTime + description: Start of time range (Unix timestamp in ms or ISO 8601) + schema: + type: string + required: false + - in: query + name: endTime + description: End of time range (Unix timestamp in ms or ISO 8601) + schema: + type: string + required: false + - in: query + name: endpointType + description: Filter by endpoint type (agent or user) + schema: + type: string + required: false + - in: query + name: resourceType + description: Filter by resource type (agent, microservice, etc.) + schema: + type: string + required: false + - in: query + name: status + description: Filter by status (SUCCESS or FAILED) + schema: + type: string + required: false + - in: query + name: method + description: Filter by HTTP/WS method(s) schema: type: array items: @@ -7495,6 +8372,10 @@ tags: description: Manage your services - name: VolumeMounts description: Manage your volume mounts + - name: Models + description: Manage your AI Model Catalogs + - name: RuntimeClasses + description: Manage your runtime classes - name: ConfigMap description: Manage your config maps - name: Events @@ -7592,12 +8473,6 @@ components: $ref: "#/components/schemas/CreateUpdateCatalogItemRequestBody" description: Microservice Catalog Item Info required: true - HalInfo: - content: - application/json: - schema: - $ref: "#/components/schemas/HalInfo" - required: true ApplicationTemplateCreateRequest: content: application/json: @@ -7781,8 +8656,19 @@ components: type: string value: type: string + ApplicationSpecBody: + type: object + properties: + microservices: + type: array + items: + $ref: "#/components/schemas/NewMicroserviceRequest" + natsConfig: + $ref: "#/components/schemas/NatsConfigObject" ApplicationCreateRequest: type: object + required: + - name properties: name: type: string @@ -7790,80 +8676,47 @@ components: type: string isActivated: type: boolean + isSystem: + type: boolean microservices: type: array items: - type: object - properties: - name: - type: string - config: - type: string - annotations: - type: string - catalogItemId: - type: integer - registryId: - type: integer - images: - type: array - items: - properties: - containerImage: - type: string - archId: - type: string - iofogUuid: - type: string - hostNetworkMode: - type: boolean - isPrivileged: - type: boolean - logSize: - type: number - rebuild: - type: boolean - volumeMappings: - type: array - items: - $ref: "#/components/schemas/VolumeMapping" - envs: - type: array - items: - type: object - properties: - key: - type: string - value: - type: string - runAsUser: - type: string - platform: - type: string - runtime: - type: string - cdiDevices: - type: array - items: - type: string - capAdd: - type: array - items: - type: string - capDrop: - type: array - items: - type: string - cmd: - type: array - items: - type: string - ports: - type: array - items: - $ref: "#/components/schemas/PortMappingsRequest" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" + $ref: "#/components/schemas/NewMicroserviceRequest" + description: >- + Nested microservice objects use the same schema as POST /microservices. + The application field on each item is set from the parent application name. + natsConfig: + $ref: "#/components/schemas/NatsConfigObject" + ApplicationUpdateRequest: + type: object + properties: + name: + type: string + description: Application resource name is immutable; must match path name if provided. + description: + type: string + isActivated: + type: boolean + isSystem: + type: boolean + microservices: + type: array + items: + $ref: "#/components/schemas/NewMicroserviceRequest" + natsConfig: + $ref: "#/components/schemas/NatsConfigObject" + ApplicationPatchRequest: + type: object + properties: + name: + type: string + description: Application resource name is immutable; must match path name if provided. + description: + type: string + isActivated: + type: boolean + isSystem: + type: boolean natsConfig: $ref: "#/components/schemas/NatsConfigObject" ApplicationCreateResponse: @@ -7889,7 +8742,7 @@ components: microservices: type: array items: - $ref: "#/components/schemas/GetMicroservicesResponse" + $ref: "#/components/schemas/MicroserviceInfoResponse" ApplicationListResponse: type: object properties: @@ -7916,86 +8769,7 @@ components: defaultValue: type: string application: - type: object - properties: - microservices: - type: array - items: - type: object - properties: - name: - type: string - config: - type: string - annotations: - type: string - catalogItemId: - type: integer - registryId: - type: integer - images: - type: array - items: - properties: - containerImage: - type: string - archId: - type: string - iofogUuid: - type: string - agentName: - type: string - hostNetworkMode: - type: boolean - isPrivileged: - type: boolean - logSize: - type: number - rebuild: - type: boolean - volumeMappings: - type: array - items: - $ref: "#/components/schemas/VolumeMapping" - envs: - type: array - items: - type: object - properties: - key: - type: string - value: - type: string - runAsUser: - type: number - platform: - type: string - runtime: - type: string - cdiDevices: - type: array - items: - type: string - capAdd: - type: array - items: - type: string - capDrop: - type: array - items: - type: string - cmd: - type: array - items: - type: string - ports: - type: array - items: - $ref: "#/components/schemas/PortMappingsRequest" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" + $ref: "#/components/schemas/ApplicationSpecBody" ApplicationTemplateCreateResponse: type: object properties: @@ -8020,14 +8794,7 @@ components: description: type: string application: - type: object - properties: - microservices: - type: array - items: - $ref: "#/components/schemas/GetMicroservicesResponse" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" + $ref: "#/components/schemas/ApplicationSpecBody" ApplicationTemplateListResponse: type: object properties: @@ -8190,10 +8957,6 @@ components: type: number logDirectory: type: string - bluetoothEnabled: - type: boolean - abstractedHardwareEnabled: - type: boolean version: type: string isReadyToUpgrade: @@ -8204,8 +8967,6 @@ components: type: number changeFrequency: type: number - deviceScanFrequency: - type: number tunnel: type: string watchdogEnabled: @@ -8270,6 +9031,25 @@ components: type: array items: type: string + runtimeClasses: + type: string + description: >- + JSON string of applied RuntimeClasses + (array of { name, handler, source }). Empty array when none. + availableCdiDevices: + type: string + description: JSON string of unique sorted fully-qualified CDI names. + modelStatus: + type: string + description: >- + JSON string of local and managed model status items. Parse for + display. Managed rows include uuid; local rows omit uuid. + activeModels: + type: integer + description: Count of managed fleet models only. + modelLastUpdate: + type: integer + description: Unix seconds; 0 when the managed list is empty. platformStatus: description: Present on GET single fog only; omitted on list responses allOf: @@ -8368,19 +9148,9 @@ components: type: number default: 20 format: seconds - deviceScanFrequency: - type: number - default: 20 - format: seconds - bluetoothEnabled: - type: boolean - default: false watchdogEnabled: type: boolean default: true - abstractedHardwareEnabled: - type: boolean - default: false archId: type: number minimum: 0 @@ -8520,9 +9290,9 @@ components: type: boolean deleteNode: type: boolean - microservicesList: + microserviceList: type: boolean - microservicesConfig: + microserviceConfig: type: boolean registries: type: boolean @@ -8532,6 +9302,21 @@ components: type: boolean routerChanged: type: boolean + volumeMounts: + type: boolean + execSessions: + type: boolean + microserviceLogs: + type: boolean + fogLogs: + type: boolean + models: + type: boolean + runtimeClasses: + type: boolean + microserviceModels: + type: boolean + description: Catalog-only refresh; agent uses GET microservices (no dedicated GET). IOFogNodeConfig: type: object properties: @@ -8565,8 +9350,6 @@ components: type: number changeFrequency: type: number - deviceScanFrequency: - type: number watchdogEnabled: type: boolean latitude: @@ -8616,8 +9399,6 @@ components: type: number changeFrequency: type: number - deviceScanFrequency: - type: number watchdogEnabled: type: boolean latitude: @@ -8710,6 +9491,35 @@ components: type: number gpsStatus: type: string + runtimeClasses: + type: array + description: Applied RuntimeClasses only, sorted by name. Docker/podman send []. + items: + type: object + properties: + name: + type: string + handler: + type: string + source: + type: string + enum: + - local + - managed + availableCdiDevices: + type: array + items: + type: string + description: Unique sorted fully-qualified CDI names (nvidia.com/gpu=0). + modelStatus: + type: string + description: JSON string of local and managed model status items. + activeModels: + type: integer + description: Count of managed fleet models only. + modelLastUpdate: + type: integer + description: Unix seconds; 0 when the managed list is empty. IOFogNodeTunnelConfigResponse: type: object properties: @@ -8793,6 +9603,43 @@ components: type: array items: type: string + description: Alias of commands. commands wins when both are present. + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean + workingDir: + type: string + cpus: + type: number + memoryReservation: + type: integer + memorySwap: + type: integer + shmSize: + type: integer + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + models: + $ref: "#/components/schemas/MicroserviceCatalog" idConsumer: type: boolean isRouter: @@ -9077,8 +9924,25 @@ components: type: string userEmail: type: string + type: + type: string + enum: + - oci + - hf + default: oci + ca: + type: string + description: Optional base64-encoded PEM CA bundle + insecure: + type: boolean + default: false RegistryBody: type: object + description: >- + type defaults to oci. Microservice and catalog-item registryId must be + type oci. Hugging Face registries are for fleet models. ca is additional + trust and does not replace system CAs. insecure true allows http and + skips TLS verify. Image-pull ca/insecure apply on edgelet only. properties: url: type: string @@ -9090,6 +9954,18 @@ components: type: string email: type: string + type: + type: string + enum: + - oci + - hf + default: oci + ca: + type: string + description: Optional base64-encoded PEM CA bundle + insecure: + type: boolean + default: false ActionBody: type: object properties: @@ -9202,16 +10078,33 @@ components: type: string name: type: string + application: + type: string + flowId: + type: integer + description: Deprecated alias of applicationId for API retrocompatibility. config: type: string annotations: type: string + catalogItemId: + type: integer + registryId: + type: integer hostNetworkMode: type: boolean isPrivileged: type: boolean - logLimit: + logSize: type: number + rebuild: + type: boolean + isActivated: + type: boolean + schedule: + type: integer + minimum: 0 + maximum: 100 images: type: array items: @@ -9219,24 +10112,43 @@ components: containerImage: type: string archId: - type: string + type: integer iofogUuid: type: string - envs: + env: type: array items: - type: object - properties: - key: - type: string - value: - type: string + $ref: "#/components/schemas/MicroserviceEnvVar" + extraHosts: + type: array + items: + $ref: "#/components/schemas/MicroserviceExtraHost" runAsUser: type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean platform: type: string runtime: - type: string + type: string + pidMode: + type: string + ipcMode: + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer + cpus: + type: number + memoryReservation: + type: integer + memorySwap: + type: integer + shmSize: + type: integer cdiDevices: type: array items: @@ -9253,6 +10165,16 @@ components: type: array items: type: string + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + workingDir: + type: string volumeMappings: type: array items: @@ -9261,6 +10183,8 @@ components: type: array items: $ref: "#/components/schemas/PortMappingsResponse" + healthCheck: + $ref: "#/components/schemas/MicroserviceHealthCheck" natsAccess: type: boolean natsConfig: @@ -9280,8 +10204,85 @@ components: type: number containerId: type: string + podId: + type: string + description: >- + Pause/sandbox id on edgelet. Same as containerId on docker/podman + when set. Omitted when unknown. + execStatus: + type: object + models: + $ref: "#/components/schemas/MicroserviceCatalog" + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + serviceAccount: + type: object + nullable: true + properties: + roleRef: + $ref: "#/components/schemas/RoleRef" + additionalProperties: false + MicroserviceEnvVar: + type: object + required: + - key + properties: + key: + type: string + value: + type: string + valueFromSecret: + type: string + description: Format secret-name/key + valueFromConfigMap: + type: string + description: Format configmap-name/key + MicroserviceExtraHost: + type: object + required: + - name + - address + properties: + name: + type: string + address: + type: string + value: + type: string + description: Present on GET responses when address is a template expression. + MicroserviceHealthCheck: + type: object + required: + - test + properties: + test: + type: array + items: + type: string + interval: + type: integer + timeout: + type: integer + startPeriod: + type: integer + startInterval: + type: integer + retries: + type: integer NewMicroserviceRequest: type: object + required: + - name properties: name: type: string @@ -9291,14 +10292,18 @@ components: type: string catalogItemId: type: integer + registryId: + type: integer images: type: array + maxItems: 4 + minItems: 1 items: properties: containerImage: type: string archId: - type: string + type: integer application: type: string iofogUuid: @@ -9310,7 +10315,18 @@ components: isPrivileged: type: boolean logSize: - type: number + type: integer + minimum: 0 + rebuild: + type: boolean + schedule: + type: integer + minimum: 0 + maximum: 100 + pidMode: + type: string + ipcMode: + type: string volumeMappings: type: array items: @@ -9318,13 +10334,39 @@ components: ports: type: array items: - $ref: "#/components/schemas/PortMappingsRequest" + $ref: "#/components/schemas/PortMappingsRequest" + env: + type: array + items: + $ref: "#/components/schemas/MicroserviceEnvVar" + extraHosts: + type: array + items: + $ref: "#/components/schemas/MicroserviceExtraHost" runAsUser: type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean platform: type: string runtime: - type: string + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer + cpus: + type: number + memoryReservation: + type: integer + minimum: 1 + memorySwap: + type: integer + shmSize: + type: integer + minimum: 1 cdiDevices: type: array items: @@ -9341,8 +10383,40 @@ components: type: array items: type: string + description: Alias of commands. commands wins when both are present. + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + workingDir: + type: string + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + healthCheck: + $ref: "#/components/schemas/MicroserviceHealthCheck" + models: + $ref: "#/components/schemas/MicroserviceCatalog" + template: + $ref: "#/components/schemas/MicroserviceTemplateRef" natsConfig: $ref: "#/components/schemas/NatsConfigObject" + natsEnabled: + type: boolean + description: YAML alias for natsConfig.natsAccess serviceAccount: type: object properties: @@ -9368,26 +10442,62 @@ components: type: boolean isPrivileged: type: boolean - logLimit: - type: number + logSize: + type: integer + minimum: 0 + schedule: + type: integer + minimum: 0 + maximum: 100 + pidMode: + type: string + ipcMode: + type: string volumeMappings: type: array items: $ref: "#/components/schemas/VolumeMapping" images: type: array + maxItems: 4 items: properties: containerImage: type: string archId: - type: string + type: integer + env: + type: array + items: + $ref: "#/components/schemas/MicroserviceEnvVar" + extraHosts: + type: array + items: + $ref: "#/components/schemas/MicroserviceExtraHost" runAsUser: type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean platform: type: string runtime: - type: string + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer + cpus: + type: number + memoryReservation: + type: integer + minimum: 1 + memorySwap: + type: integer + shmSize: + type: integer + minimum: 1 cdiDevices: type: array items: @@ -9404,8 +10514,40 @@ components: type: array items: type: string + description: Alias of commands. commands wins when both are present. + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + workingDir: + type: string + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + healthCheck: + $ref: "#/components/schemas/MicroserviceHealthCheck" + models: + $ref: "#/components/schemas/MicroserviceCatalog" + template: + $ref: "#/components/schemas/MicroserviceTemplateRef" natsConfig: $ref: "#/components/schemas/NatsConfigObject" + natsEnabled: + type: boolean + description: YAML alias for natsConfig.natsAccess serviceAccount: type: object properties: @@ -10165,13 +11307,6 @@ components: type: string description: Target semver for upgrade or rollback pattern: ^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:[+]([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ - HalInfo: - type: object - required: - - info - properties: - info: - type: string PostTrackingRequest: type: array items: @@ -10685,6 +11820,278 @@ components: minItems: 1 required: - fogUuids + FogLinkRequest: + type: object + properties: + fogUuids: + type: array + items: + type: string + minItems: 1 + required: + - fogUuids + FogLinkSet: + type: object + properties: + fogUuids: + type: array + items: + type: string + Model: + type: object + properties: + uuid: + type: string + name: + type: string + repo: + type: string + description: Upstream path without host + revision: + type: string + description: Pin. Empty means latest (OCI) or main (HF). + registryId: + type: integer + files: + type: array + items: + type: string + description: Hugging Face only. Ignored for OCI registries. + format: + type: string + enum: + - gguf + - safetensors + - onnx + - pytorch + - tensorrt + - unknown + ModelsListResponse: + type: object + properties: + models: + type: array + items: + $ref: "#/components/schemas/Model" + AgentModelsListResponse: + type: object + properties: + models: + type: array + items: + $ref: "#/components/schemas/Model" + ModelCreate: + type: object + required: + - name + - repo + - registryId + properties: + name: + type: string + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + enum: + - gguf + - safetensors + - onnx + - pytorch + - tensorrt + - unknown + ModelUpdate: + type: object + properties: + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + enum: + - gguf + - safetensors + - onnx + - pytorch + - tensorrt + - unknown + RuntimeClass: + type: object + properties: + name: + type: string + handler: + type: string + RuntimeClassesListResponse: + type: object + properties: + runtimeClasses: + type: array + items: + $ref: "#/components/schemas/RuntimeClass" + AgentRuntimeClassesListResponse: + type: object + properties: + runtimeClasses: + type: array + items: + $ref: "#/components/schemas/RuntimeClass" + RuntimeClassCreate: + type: object + required: + - name + - handler + properties: + name: + type: string + handler: + type: string + RuntimeClassUpdate: + type: object + properties: + handler: + type: string + MicroserviceCatalogItem: + type: object + required: + - name + properties: + name: + type: string + description: Managed fleet model name. Never uuid or a host path. + MicroserviceCatalog: + type: object + properties: + bindPath: + type: string + description: Required when items is non-empty. Absolute container path. + permissions: + type: string + enum: + - ro + - rw + default: ro + items: + type: array + items: + $ref: "#/components/schemas/MicroserviceCatalogItem" + ContainerDevice: + type: object + required: + - hostPath + - containerPath + properties: + hostPath: + type: string + containerPath: + type: string + permissions: + type: string + ContainerTmpfs: + type: object + required: + - containerPath + properties: + containerPath: + type: string + size: + type: integer + mode: + type: string + MicroserviceTemplateVariable: + type: object + required: + - key + properties: + key: + type: string + description: + type: string + defaultValue: {} + MicroserviceTemplate: + type: object + properties: + name: + type: string + description: + type: string + variables: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplateVariable" + microservice: + type: object + description: Microservice spec without instance identity fields. + MicroserviceTemplatesListResponse: + type: object + properties: + microserviceTemplates: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplate" + MicroserviceTemplateCreate: + type: object + required: + - name + - microservice + properties: + name: + type: string + description: + type: string + variables: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplateVariable" + microservice: + type: object + MicroserviceTemplateUpdate: + type: object + properties: + description: + type: string + variables: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplateVariable" + microservice: + type: object + MicroserviceTemplateRef: + type: object + required: + - name + properties: + name: + type: string + variables: + description: Map of key to value, or array of { key, value }. + oneOf: + - type: object + additionalProperties: true + - type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key RbacRule: type: object properties: diff --git a/src/config/rbac-resources.yaml b/src/config/rbac-resources.yaml index cd2db3be..e56cfeb3 100644 --- a/src/config/rbac-resources.yaml +++ b/src/config/rbac-resources.yaml @@ -55,6 +55,10 @@ resources: methods: DELETE: [patch] resourceNameParam: uuid + - path: /api/v3/microservices/:uuid/models + methods: + PATCH: [patch] + resourceNameParam: uuid - path: /api/v3/microservices/:uuid/start methods: PATCH: [patch] @@ -138,14 +142,6 @@ resources: methods: POST: [patch] resourceNameParam: uuid - - path: /api/v3/iofog/:uuid/hal/hw - methods: - GET: [get] - resourceNameParam: uuid - - path: /api/v3/iofog/:uuid/hal/usb - methods: - GET: [get] - resourceNameParam: uuid - path: /api/v3/iofog/:uuid/prune methods: POST: [patch] @@ -529,6 +525,84 @@ resources: DELETE: [patch] resourceNameParam: name + # Fleet models + models: + basePath: /api/v3/models + routes: + - path: /api/v3/models + methods: + GET: [list] + POST: [create] + - path: /api/v3/models/yaml + methods: + POST: [create] + - path: /api/v3/models/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/models/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + - path: /api/v3/models/:name/link + methods: + GET: [get] + POST: [patch] + DELETE: [patch] + resourceNameParam: name + + # Runtime classes + runtimeClasses: + basePath: /api/v3/runtimeClasses + routes: + - path: /api/v3/runtimeClasses + methods: + GET: [list] + POST: [create] + - path: /api/v3/runtimeClasses/yaml + methods: + POST: [create] + - path: /api/v3/runtimeClasses/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/runtimeClasses/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + - path: /api/v3/runtimeClasses/:name/link + methods: + GET: [get] + POST: [patch] + DELETE: [patch] + resourceNameParam: name + + # Microservice templates + microserviceTemplates: + basePath: /api/v3/microserviceTemplates + routes: + - path: /api/v3/microserviceTemplates + methods: + GET: [list] + POST: [create] + - path: /api/v3/microserviceTemplates/yaml + methods: + POST: [create] + - path: /api/v3/microserviceTemplates/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/microserviceTemplates/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + # Tunnels tunnels: basePath: /api/v3/iofog @@ -644,6 +718,12 @@ resources: - path: /api/v3/agent/volumeMounts methods: GET: [list] + - path: /api/v3/agent/models + methods: + GET: [list] + - path: /api/v3/agent/runtimeClasses + methods: + GET: [list] - path: /api/v3/agent/microservices methods: GET: [list] @@ -660,12 +740,6 @@ resources: - path: /api/v3/agent/version methods: GET: [get] - - path: /api/v3/agent/hal/hw - methods: - PUT: [update] - - path: /api/v3/agent/hal/usb - methods: - PUT: [update] - path: /api/v3/agent/delete-node methods: DELETE: [delete] diff --git a/src/config/rbac-system-roles.js b/src/config/rbac-system-roles.js index 3dbe4ee1..3fc0170d 100644 --- a/src/config/rbac-system-roles.js +++ b/src/config/rbac-system-roles.js @@ -40,7 +40,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'systemMicroservices', 'fogs', 'applications', 'systemApplications', 'applicationTemplates', 'services', 'router', 'networkTopology', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'tunnels', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'events', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'execSessions', 'systemExecSessions', 'logs', 'systemLogs'], + resources: ['microservices', 'systemMicroservices', 'fogs', 'applications', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'runtimeClasses', 'tunnels', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'events', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'execSessions', 'systemExecSessions', 'logs', 'systemLogs'], verbs: ['*'] }, { @@ -62,7 +62,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'applications', 'applicationTemplates', 'services', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'certificates', 'capabilities', 'serviceAccounts', 'controller', 'execSessions', 'logs'], + resources: ['microservices', 'applications', 'applicationTemplates', 'microserviceTemplates', 'services', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'runtimeClasses', 'certificates', 'capabilities', 'serviceAccounts', 'controller', 'execSessions', 'logs'], verbs: ['get', 'list', 'create', 'update', 'patch', 'delete'] }, { @@ -84,7 +84,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'fogs', 'applications', 'systemMicroservices', 'systemApplications', 'applicationTemplates', 'services', 'router', 'networkTopology', 'natsOperator', 'natsBootstrap', 'natsHub', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'roles', 'roleBindings'], + resources: ['microservices', 'fogs', 'applications', 'systemMicroservices', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsOperator', 'natsBootstrap', 'natsHub', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'runtimeClasses', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'roles', 'roleBindings'], verbs: ['get', 'list'] } ] diff --git a/test/src/config/rbac-audit.test.js b/test/src/config/rbac-audit.test.js new file mode 100644 index 00000000..d53c3be4 --- /dev/null +++ b/test/src/config/rbac-audit.test.js @@ -0,0 +1,44 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') +const yaml = require('js-yaml') +const { auditRbac } = require('../../../scripts/rbac-audit') + +const CATALOG_PATH = path.join(__dirname, '../../../src/config/rbac-resources.yaml') + +describe('RBAC route catalog', () => { + it('registers every live Express route in rbac-resources.yaml', () => { + const result = auditRbac() + expect(result.gaps, JSON.stringify(result.gaps, null, 2)).to.deep.equal([]) + expect(result.banned).to.deep.equal([]) + expect(result.missing).to.deep.equal([]) + }) + + it('registers fleet model, RuntimeClass, and microservice template routes', () => { + const catalog = yaml.load(fs.readFileSync(CATALOG_PATH, 'utf8')) + + expect(catalog.resources.models).to.be.an('object') + expect(catalog.resources.runtimeClasses).to.be.an('object') + expect(catalog.resources.microserviceTemplates).to.be.an('object') + expect(catalog.resources.microservices.routes.some((route) => { + return route.path === '/api/v3/microservices/:uuid/models' + })).to.equal(true) + expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/models')).to.equal(true) + expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/runtimeClasses')).to.equal(true) + }) + + it('does not leave stale yaml entries for fleet model or RuntimeClass routes', () => { + const result = auditRbac() + const fleetOrphans = result.orphans.filter((orphan) => { + return orphan.resource === 'models' || + orphan.resource === 'runtimeClasses' || + orphan.resource === 'microserviceTemplates' || + orphan.path.includes('/agent/models') || + orphan.path.includes('/agent/runtimeClasses') || + orphan.path.includes('/microservices/:uuid/models') + }) + expect(fleetOrphans, JSON.stringify(fleetOrphans, null, 2)).to.deep.equal([]) + }) +}) From b2f94a1358cd6b14f15fe97caa6b7d04223f6fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:51:33 +0300 Subject: [PATCH 11/55] Cover fleet model and RuntimeClass flows and record the unreleased notes. Add integration coverage and grep gates so hardware-inventory remnants and leaked planning tokens stay out of application code. --- CHANGELOG.md | 26 + package-lock.json | 22 +- package.json | 11 +- .../helpers/transaction-grep-gates.test.js | 66 +++ .../fleet-models-runtime-sqlite.test.js | 448 ++++++++++++++++++ 5 files changed, 557 insertions(+), 16 deletions(-) create mode 100644 test/src/integration/fleet-models-runtime-sqlite.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index cea27334..b90ab6b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,32 @@ # Changelog +## [Unreleased] + +Fleet models, RuntimeClass, registry extensions, microservice catalog PATCH, MicroserviceTemplate, and HAL/BLE removal for Edgelet v1.1+ handoff. Targets **v3.9.0** schema (user tags). + +### Added + +- **Fleet models API** — `GET/POST/PATCH/DELETE /api/v3/models`, YAML create/upsert, and volumeMount-style `GET/POST/DELETE …/models/:name/link`; server-generated uuid; agent `GET /api/v3/agent/models` returns linked rows only. +- **RuntimeClass API** — `GET/POST/PATCH/DELETE /api/v3/runtimeClasses`, YAML, link endpoints (edgelet fogs only), and agent `GET /api/v3/agent/runtimeClasses`. +- **Registry `type` / `ca` / `insecure`** — Hub system registry by Hugging Face url+`hf`; MS and catalog image pulls remain **oci-only**. +- **`PATCH /api/v3/microservices/:uuid/models`** — `microserviceModels` change flag and rebuild rules for bind path / permissions / empty↔non-empty catalog transitions. +- **Microservice container fields** — commands/cmd alias, CDI devices, capabilities, health check, and related agent wire fields. +- **MicroserviceTemplate** — CRUD + YAML; deploy overlay via `POST /microservices` + `template`. +- **Agent status ingest** — `modelStatus`, `activeModels`, `modelLastUpdate`, `runtimeClasses`, `availableCdiDevices`, `podId`; getChanges flags `models`, `runtimeClasses`, `microserviceModels`. +- **v3.9.0 schema** — `Models`, `RuntimeClasses`, link tables, registry columns, template tables. Fresh install uses baseline 3.9.0 migrate+seed; upgrade from 3.8.0 applies incremental 3.9.0 only. + +### Changed + +- **MS deploy** — auto-links referenced fleet models and RuntimeClass on edgelet agents; validates runtime against `availableRuntimes`. +- **RBAC + swagger** — models, runtimeClasses, microserviceTemplates, and agent model/runtime routes documented and catalogued. + +### Removed + +- **HAL/BLE provisioning** — user HAL/USB routes, RESTBlue/HAL system catalog items, fog `bluetooth`/`hal` flags, and `deviceScanFrequency` from agent config wire (Edge Guard retained). +- **Agent HAL routes** — hardware inventory PUT/GET paths under `/api/v3/agent/hal/*`. + + ## [v3.8.3-rc.3] - September 2026 Additional security dependency patches and dev tooling refresh. diff --git a/package-lock.json b/package-lock.json index a837a696..5fde2339 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "controller", - "version": "3.8.3", + "version": "3.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "controller", - "version": "3.8.3", + "version": "3.9.0", "hasInstallScript": true, "license": "EPL-2.0", "dependencies": { @@ -44,10 +44,10 @@ "helmet": "7.1.0", "is-elevated": "3.0.0", "jose": "^4.15.9", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "jsonschema": "1.5.0", "moment": "2.30.1", - "multer": "2.2.0", + "multer": "2.3.0", "mysql2": "3.23.1", "nconf": "0.12.1", "node-fetch-npm": "^2.0.4", @@ -78,7 +78,7 @@ "chai-as-promised": "7.1.2", "chai-http": "4.4.0", "eslint": "8.57.1", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "mocha": "11.7.6", "mocha-junit-reporter": "2.2.1", "nyc": "18.0.0", @@ -8626,9 +8626,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "funding": [ { "type": "github", @@ -9628,9 +9628,9 @@ "license": "MIT" }, "node_modules/multer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.2.0.tgz", - "integrity": "sha512-6rdyFg2kLrMh9Jee7/BMPuV9lEAd7lLW2YUpF9/YxR7njyoUwwQ0ZPh3TaIY50Sw6vlyD2HW3wGOkTS4P79xrQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.3.0.tgz", + "integrity": "sha512-cjNbm3sttszgZeGfJR124D+jFEfkXCVAsoPBmFn9X7UxmDSFHWqE2CoEj0vrmSpuAFnqWR1Szcm9QTsiHr60Xw==", "license": "MIT", "dependencies": { "append-field": "^1.0.0", diff --git a/package.json b/package.json index 4aa29087..8409d2fa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "controller", - "version": "3.8.3", + "version": "3.9.0", "description": "A cloud-native operations controller for managing edge computing workloads, edgelet nodes, and deployments across Eclipse IoFog and Datasance PoT.", "main": "./src/main.js", "author": "Eclipse ioFog Project", @@ -58,6 +58,7 @@ "test:all": "node scripts/run-test.js test-all", "test:k8s-client": "node scripts/run-test.js test test/integration/k8s-client-integration.test.js", "test:integration:first-fog": "RUN_INTEGRATION=1 node scripts/run-test.js test test/src/integration/first-fog-reconcile-sqlite.test.js", + "test:integration:fleet-models": "RUN_INTEGRATION=1 node scripts/run-test.js test test/src/integration/fleet-models-runtime-sqlite.test.js", "test:ws-load": "node test/load/ws-pairing-load.js", "test:load:tx": "node test/load/transaction-safety-load.js --fogs 200 --soak-minutes 5", "precli-tests": "npm run lint", @@ -109,10 +110,10 @@ "helmet": "7.1.0", "is-elevated": "3.0.0", "jose": "^4.15.9", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "jsonschema": "1.5.0", "moment": "2.30.1", - "multer": "2.2.0", + "multer": "2.3.0", "mysql2": "3.23.1", "@nats-io/transport-node": "^3.4.0", "nconf": "0.12.1", @@ -141,7 +142,7 @@ "chai-as-promised": "7.1.2", "chai-http": "4.4.0", "eslint": "8.57.1", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "mocha": "11.7.6", "mocha-junit-reporter": "2.2.1", "nyc": "18.0.0", @@ -163,7 +164,7 @@ "overrides": { "tar": "^7.5.19", "protobufjs": "^7.6.5", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "brace-expansion": "5.0.9", "uuid": "11.1.1", "diff": "8.0.3", diff --git a/test/src/helpers/transaction-grep-gates.test.js b/test/src/helpers/transaction-grep-gates.test.js index 05ba022e..d8f13ed7 100644 --- a/test/src/helpers/transaction-grep-gates.test.js +++ b/test/src/helpers/transaction-grep-gates.test.js @@ -29,6 +29,21 @@ function grepSrc (pattern, extraArgs = [], searchPath = 'src/') { } } +function grepPaths (pattern, extraArgs, searchPaths) { + const hits = [] + for (const searchPath of searchPaths) { + const resolved = path.join(REPO_ROOT, searchPath) + if (!fs.existsSync(resolved)) { + continue + } + const match = grepSrc(pattern, extraArgs, searchPath) + if (match) { + hits.push(match) + } + } + return hits.join('\n') +} + /** * Plan 19-I-C: K8s I/O must run outside runInTransaction callback bodies. * Allowed external helpers (called after tx commit / between phases): @@ -167,6 +182,17 @@ describe('grep gates', () => { expect(volumeMountSource).to.not.match(/removeVolumeMount\(volumeMount\.uuid, transaction\)/) }) + it('passes Sequelize transaction inside options for model association calls', () => { + const modelSource = fs.readFileSync( + path.join(REPO_ROOT, 'src/services/model-service.js'), + 'utf8' + ) + expect(modelSource).to.match(/getFogs\(\{ transaction \}\)/) + expect(modelSource).to.match(/addModel\(model, \{ transaction \}\)/) + expect(modelSource).to.match(/removeModel\(model, \{ transaction \}\)/) + expect(modelSource).to.match(/hasModel\(model, \{ transaction \}\)/) + }) + it('keeps vault HTTP out of secret/configmap/registry transaction bodies', () => { const secretSource = fs.readFileSync( path.join(REPO_ROOT, 'src/services/secret-service.js'), @@ -261,6 +287,46 @@ describe('grep gates', () => { expect(grepSrc('ChangeTrackingService', [], 'src/services/registry-service.js')).to.equal('') }) + it('has zero plan or RFC rule tokens in application code, tests, swagger, migrations, and seeders', () => { + const codeHits = grepPaths( + 'R1[5-7][0-9]|plan.?23|BR-', + ['-E', '-i', '--include=*.js', '--exclude=transaction-grep-gates.test.js'], + ['src/', 'test/'] + ) + const sqlHits = grepPaths( + 'R1[5-7][0-9]|plan.?23|BR-', + ['-E', '-i', '--include=*.sql'], + ['src/data/migrations/', 'src/data/seeders/'] + ) + const swaggerHits = grepSrc( + 'R1[5-7][0-9]|plan.?23|BR-', + ['-E', '-i'], + 'docs/swagger.yaml' + ) + expect([codeHits, sqlHits, swaggerHits].filter(Boolean).join('\n')).to.equal('') + }) + + it('has zero hardware inventory and bluetooth catalog leftovers in application JS', () => { + const hits = grepSrc( + 'deviceScanFrequency|abstractedHardwareEnabled|bluetoothEnabled|getHalCatalogItem|getBluetoothCatalogItem|RESTBlue', + ['-E', '--exclude-dir=cli'] + ) + expect(hits).to.equal('') + }) + + it('does not register hardware inventory HTTP routes', () => { + expect(grepSrc('/hal/', [], 'src/routes/')).to.equal('') + const yamlHits = grepSrc('/agent/hal/|/iofog/.*/hal/', [ + '-E', + '--include=*.yaml' + ], 'src/config') + expect(yamlHits).to.equal('') + }) + + it('keeps Edge Guard frequency on fog config', () => { + expect(grepSrc('edgeGuardFrequency')).to.not.equal('') + }) + it('keeps MicroserviceManager.findAllWithStatuses fan-out out of catalog/registry propagation paths', () => { const catalogSource = fs.readFileSync( path.join(REPO_ROOT, 'src/services/catalog-service.js'), diff --git a/test/src/integration/fleet-models-runtime-sqlite.test.js b/test/src/integration/fleet-models-runtime-sqlite.test.js new file mode 100644 index 00000000..1a7a77f5 --- /dev/null +++ b/test/src/integration/fleet-models-runtime-sqlite.test.js @@ -0,0 +1,448 @@ +'use strict' + +/** + * Fleet models, RuntimeClass, and registry gates — sqlite integration paths. + * + * Skipped in default `npm test` unless RUN_INTEGRATION=1 (full DB migrate + init). + * + * RUN_INTEGRATION=1 npm run test:integration:fleet-models + */ + +describe('fleet models and runtime sqlite integration', function () { + this.timeout(60000) + + let harness + let runTx + let systemFog + let testCounter = 0 + + function uniqueSuffix () { + testCounter += 1 + return `${Date.now()}-${testCounter}` + } + + before(function () { + if (process.env.RUN_INTEGRATION !== '1') { + this.skip() + } + }) + + before(async function () { + const { + createFirstFogSqliteHarness, + driveReconcileUntilReady + } = require('../../support/first-fog-sqlite-harness') + + harness = await createFirstFogSqliteHarness() + + const { runInTransaction, PRIORITY_INTERACTIVE } = require('../../../src/helpers/transaction-runner') + runTx = (fn, label) => runInTransaction(fn, { + priority: PRIORITY_INTERACTIVE, + label: label || 'integration.fleetModelsRuntime' + }) + + systemFog = await createSystemFog('shared') + + const FogPlatformStatusManager = require('../../../src/data/managers/fog-platform-status-manager') + const { drainOnce } = require('../../../src/jobs/reconcile-outbox-drainer-job') + const PlatformReconcileWorkerJob = require('../../../src/jobs/platform-reconcile-worker-job') + await driveReconcileUntilReady(systemFog.uuid, { + drainOnce, + processNextFogTask: PlatformReconcileWorkerJob.processNextFogTask, + getStatus: (uuid) => FogPlatformStatusManager.getParsedStatus(uuid) + }) + }) + + after(async function () { + if (harness) { + await harness.teardown() + } + }) + + async function createSystemFog (suffix) { + return runTx( + (transaction) => require('../../../src/services/iofog-service').createFogEndPoint({ + name: `hub-fleet-${suffix}`, + host: '127.0.0.1', + archId: 1, + containerEngine: 'edgelet' + }, false, transaction), + 'integration.createSystemFog' + ) + } + + async function createEdgeFog (suffix) { + return runTx( + (transaction) => require('../../../src/services/iofog-service').createFogEndPoint({ + name: `edge-fleet-${suffix}`, + host: '127.0.0.1', + archId: 1, + containerEngine: 'edgelet', + routerMode: 'edge', + natsMode: 'leaf', + upstreamRouters: [systemFog.uuid] + }, false, transaction), + 'integration.createEdgeFog' + ) + } + + async function createDockerFog (suffix) { + return runTx( + (transaction) => require('../../../src/services/iofog-service').createFogEndPoint({ + name: `docker-fleet-${suffix}`, + host: '127.0.0.1', + archId: 1, + containerEngine: 'docker', + routerMode: 'edge', + natsMode: 'leaf', + upstreamRouters: [systemFog.uuid] + }, false, transaction), + 'integration.createDockerFog' + ) + } + + async function loadFog (uuid) { + const FogManager = require('../../../src/data/managers/iofog-manager') + return runTx((transaction) => FogManager.findOne({ uuid }, transaction), 'integration.loadFog') + } + + it('returns linked models on agent GET after model create and fog link', async function () { + const { expect } = require('chai') + const ModelService = require('../../../src/services/model-service') + const AgentService = require('../../../src/services/agent-service') + + const suffix = uniqueSuffix() + const modelName = `test-model-${suffix}` + const edgeFog = await createEdgeFog(suffix) + + await runTx( + (transaction) => ModelService.createModelEndpoint({ + name: modelName, + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }, transaction), + 'integration.createModel' + ) + + await runTx( + (transaction) => ModelService.linkModelEndpoint(modelName, [edgeFog.uuid], transaction), + 'integration.linkModel' + ) + + const fog = await loadFog(edgeFog.uuid) + const models = await runTx( + (transaction) => AgentService.getAgentLinkedModels(fog, transaction), + 'integration.agentModels' + ) + + expect(models).to.have.length(1) + expect(models[0]).to.include({ + name: modelName, + repo: 'org/repo', + registryId: 3, + format: 'gguf' + }) + expect(models[0].files).to.eql(['file.gguf']) + }) + + it('returns linked runtime class on agent GET after link on edgelet fog', async function () { + const { expect } = require('chai') + const RuntimeClassService = require('../../../src/services/runtime-class-service') + const AgentService = require('../../../src/services/agent-service') + + const suffix = uniqueSuffix() + const edgeFog = await createEdgeFog(suffix) + const className = `spin-${suffix}` + + await runTx( + (transaction) => RuntimeClassService.createRuntimeClassEndpoint({ + name: className, + handler: 'spin' + }, transaction), + 'integration.createRuntimeClass' + ) + + await runTx( + (transaction) => RuntimeClassService.linkRuntimeClassEndpoint(className, [edgeFog.uuid], transaction), + 'integration.linkRuntimeClass' + ) + + const fog = await loadFog(edgeFog.uuid) + const runtimeClasses = await runTx( + (transaction) => AgentService.getAgentLinkedRuntimeClasses(fog, transaction), + 'integration.agentRuntimeClasses' + ) + + expect(runtimeClasses).to.eql([{ name: className, handler: 'spin' }]) + }) + + it('refuses runtime class link on docker fog with validation error', async function () { + const { expect } = require('chai') + const RuntimeClassService = require('../../../src/services/runtime-class-service') + const Errors = require('../../../src/helpers/errors') + + const suffix = uniqueSuffix() + const dockerFog = await createDockerFog(suffix) + const className = `spin-docker-${suffix}` + + await runTx( + (transaction) => RuntimeClassService.createRuntimeClassEndpoint({ + name: className, + handler: 'spin' + }, transaction), + 'integration.createRuntimeClassDocker' + ) + + await expect(runTx( + (transaction) => RuntimeClassService.linkRuntimeClassEndpoint(className, [dockerFog.uuid], transaction), + 'integration.linkRuntimeClassDocker' + )).to.be.rejectedWith(Errors.ValidationError, /edgelet/) + }) + + it('sets microserviceModels change flag and rebuild on catalog PATCH', async function () { + const { expect } = require('chai') + const ApplicationService = require('../../../src/services/application-service') + const MicroserviceService = require('../../../src/services/microservices-service') + const ModelService = require('../../../src/services/model-service') + const AgentService = require('../../../src/services/agent-service') + const MicroserviceManager = require('../../../src/data/managers/microservice-manager') + + const suffix = uniqueSuffix() + const edgeFog = await createEdgeFog(suffix) + const modelName = `catalog-model-${suffix}` + const appName = `fleet-catalog-app-${suffix}` + + await runTx( + (transaction) => ModelService.createModelEndpoint({ + name: modelName, + repo: 'org/repo', + registryId: 3, + files: ['weights.gguf'] + }, transaction), + 'integration.createCatalogModel' + ) + + await runTx( + (transaction) => ApplicationService.createApplicationEndPoint({ + name: appName, + description: 'catalog patch integration' + }, false, transaction), + 'integration.createCatalogApp' + ) + + const microservice = await runTx( + (transaction) => MicroserviceService.createMicroserviceEndPoint({ + name: `catalog-ms-${suffix}`, + application: appName, + iofogUuid: edgeFog.uuid, + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1 + }, false, transaction), + 'integration.createCatalogMs' + ) + + await runTx( + (transaction) => MicroserviceService.updateMicroserviceCatalogEndPoint(microservice.uuid, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: modelName }] + }, false, transaction), + 'integration.patchCatalog' + ) + + const updated = await runTx( + (transaction) => MicroserviceManager.findOne({ uuid: microservice.uuid }, transaction), + 'integration.loadCatalogMs' + ) + expect(updated.rebuild).to.equal(true) + + const MicroserviceModelManager = require('../../../src/data/managers/microservice-model-manager') + const MicroserviceModelItemManager = require('../../../src/data/managers/microservice-model-item-manager') + const catalogRow = await runTx( + (transaction) => MicroserviceModelManager.findOne({ microserviceUuid: microservice.uuid }, transaction), + 'integration.loadCatalogRow' + ) + const catalogItems = await runTx( + (transaction) => MicroserviceModelItemManager.findAll({ microserviceUuid: microservice.uuid }, transaction), + 'integration.loadCatalogItems' + ) + expect(catalogRow.bindPath).to.equal('/models') + expect(catalogItems.map((item) => item.name)).to.eql([modelName]) + + const fog = await loadFog(edgeFog.uuid) + const changes = await runTx( + (transaction) => AgentService.getAgentConfigChanges(fog, transaction), + 'integration.catalogChanges' + ) + expect(changes.microserviceList).to.equal(true) + }) + + it('auto-links model and runtime class rows when microservice deploy references them', async function () { + const { expect } = require('chai') + const ApplicationService = require('../../../src/services/application-service') + const MicroserviceService = require('../../../src/services/microservices-service') + const ModelService = require('../../../src/services/model-service') + const RuntimeClassService = require('../../../src/services/runtime-class-service') + const FogManager = require('../../../src/data/managers/iofog-manager') + + const suffix = uniqueSuffix() + const className = `spin-${suffix}` + const modelName = `deploy-model-${suffix}` + const appName = `fleet-deploy-app-${suffix}` + const edgeFog = await createEdgeFog(suffix) + + await runTx( + (transaction) => FogManager.update( + { uuid: edgeFog.uuid }, + { availableRuntimes: JSON.stringify([className]) }, + transaction + ), + 'integration.setAvailableRuntimes' + ) + + await runTx( + (transaction) => ModelService.createModelEndpoint({ + name: modelName, + repo: 'org/repo', + registryId: 3, + files: ['weights.gguf'] + }, transaction), + 'integration.createDeployModel' + ) + + await runTx( + (transaction) => RuntimeClassService.createRuntimeClassEndpoint({ + name: className, + handler: 'spin' + }, transaction), + 'integration.createDeployRuntimeClass' + ) + + await runTx( + (transaction) => ApplicationService.createApplicationEndPoint({ + name: appName, + description: 'deploy auto-link integration' + }, false, transaction), + 'integration.createDeployApp' + ) + + await runTx( + (transaction) => MicroserviceService.createMicroserviceEndPoint({ + name: `deploy-ms-${suffix}`, + application: appName, + iofogUuid: edgeFog.uuid, + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1, + runtime: className, + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: modelName }] + } + }, false, transaction), + 'integration.createDeployMs' + ) + + const fog = await loadFog(edgeFog.uuid) + const linkedModels = await runTx( + (transaction) => fog.getModels({ transaction }), + 'integration.loadModelLinks' + ) + const linkedRuntimeClasses = await runTx( + (transaction) => fog.getRuntimeClassLinks({ transaction }), + 'integration.loadRuntimeLinks' + ) + + expect(linkedModels).to.have.length(1) + expect(linkedModels[0].name).to.equal(modelName) + expect(linkedRuntimeClasses).to.have.length(1) + expect(linkedRuntimeClasses[0].name).to.equal(className) + }) + + it('rejects Hugging Face registry for microservice images', async function () { + const { expect } = require('chai') + const ApplicationService = require('../../../src/services/application-service') + const MicroserviceService = require('../../../src/services/microservices-service') + const AppHelper = require('../../../src/helpers/app-helper') + const ErrorMessages = require('../../../src/helpers/error-messages') + + const suffix = uniqueSuffix() + const appName = `fleet-registry-app-${suffix}` + const edgeFog = await createEdgeFog(suffix) + + await runTx( + (transaction) => ApplicationService.createApplicationEndPoint({ + name: appName, + description: 'registry gate integration' + }, false, transaction), + 'integration.createRegistryApp' + ) + + await expect(runTx( + (transaction) => MicroserviceService.createMicroserviceEndPoint({ + name: `registry-ms-${suffix}`, + application: appName, + iofogUuid: edgeFog.uuid, + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 3 + }, false, transaction), + 'integration.createRegistryMs' + )).to.be.rejectedWith(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 3)) + }) + + it('persists modelStatus from agent PUT and returns parseable JSON on user GET', async function () { + const { expect } = require('chai') + const AgentService = require('../../../src/services/agent-service') + const IofogService = require('../../../src/services/iofog-service') + + const modelStatusPayload = [{ name: 'test-model', state: 'ready' }] + const modelStatusJson = JSON.stringify(modelStatusPayload) + + await runTx( + (transaction) => AgentService.updateAgentStatus({ + daemonStatus: 'RUNNING', + warningMessage: '', + daemonOperatingDuration: 1, + daemonLastStart: 1, + memoryUsage: 1, + diskUsage: 1, + cpuUsage: 1, + memoryViolation: false, + diskViolation: false, + cpuViolation: false, + systemAvailableDisk: 1, + systemAvailableMemory: 1, + systemTotalCpu: 1, + repositoryCount: 0, + repositoryStatus: '[]', + systemTime: 1, + lastStatusTime: 1, + ipAddress: '127.0.0.1', + ipAddressExternal: '127.0.0.1', + lastCommandTime: 1, + tunnelStatus: '{}', + version: '1.1.0', + isReadyToUpgrade: false, + isReadyToRollback: false, + gpsStatus: 'OK', + microserviceStatus: '[]', + modelStatus: modelStatusJson, + activeModels: 1, + modelLastUpdate: 1710000000 + }, { uuid: systemFog.uuid }, transaction), + 'integration.agentStatusModel' + ) + + const fog = await runTx( + (transaction) => IofogService.getFogEndPoint({ uuid: systemFog.uuid }, false, transaction), + 'integration.userGetFog' + ) + + expect(fog.modelStatus).to.equal(modelStatusJson) + expect(fog.activeModels).to.equal(1) + expect(fog.modelLastUpdate).to.equal(1710000000) + expect(JSON.parse(fog.modelStatus)).to.eql(modelStatusPayload) + }) +}) From a785dda95f69a20ff337ef4d1a8eaade9b23a41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 16 Sep 2026 19:52:04 +0300 Subject: [PATCH 12/55] bump container base image digest pins. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07d61d88..818bb46b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # ioFog overrides: EDGEOPS_CONSOLE_REPO=https://github.com/eclipse-iofog/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=iofog # node:24-bookworm — pin manifest list digest for reproducible multi-arch builds -FROM node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 AS console-builder +FROM node:24-bookworm@sha256:6dac556d980b7f0e5498d08f08cee0ca67798b4ad6c23964a9214920e67758d0 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console ARG EDGEOPS_CONSOLE_VERSION=v1.0.13 @@ -28,7 +28,7 @@ RUN test -f build/index.html \ && cp -a build /tmp/console/build -FROM node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 AS builder +FROM node:24-bookworm@sha256:6dac556d980b7f0e5498d08f08cee0ca67798b4ad6c23964a9214920e67758d0 AS builder ARG PKG_VERSION @@ -48,7 +48,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds -FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:c7b8cb436feace96ca0b43eab7326394893e0ac9c2c6b46895727e3b9202fb9d +FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:0838eaa65048d6c76749b92f423ee73af7fdd72b32b365a0fdc200b3ab74f363 ARG EDGEOPS_CONSOLE_VERSION=v1.0.13 ARG IMAGE_REGISTRY From eed2224ce0e439771ef714b26fd6a38fcd3612cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Thu, 17 Sep 2026 16:07:52 +0300 Subject: [PATCH 13/55] Parse microservice template YAML with placeholders and omit unspecified container fields. --- src/services/yaml-parser-service.js | 285 +++++++++++++----- test/src/services/yaml-parser-service.test.js | 69 ++++- 2 files changed, 275 insertions(+), 79 deletions(-) diff --git a/src/services/yaml-parser-service.js b/src/services/yaml-parser-service.js index c0296fc8..cfd33962 100644 --- a/src/services/yaml-parser-service.js +++ b/src/services/yaml-parser-service.js @@ -237,23 +237,130 @@ const { mapYamlImagesToArchList } = require('../helpers/arch-images') const mapImages = (images) => mapYamlImagesToArchList(images) -const parseMicroserviceImages = async (fileImages) => { +const REGISTRY_BY_NAME = { + remote: 1, + local: 2 +} + +function isTemplatePlaceholder (value) { + return typeof value === 'string' && value.includes('{{') +} + +function asIdOrPlaceholder (value, registryByName = REGISTRY_BY_NAME) { + if (value == null || value === '') { + return undefined + } + if (typeof value === 'number' && Number.isFinite(value)) { + return value + } + if (typeof value === 'string') { + const trimmed = value.trim() + if (isTemplatePlaceholder(trimmed)) { + return trimmed + } + if (Object.hasOwn(registryByName, trimmed)) { + return registryByName[trimmed] + } + const asNumber = Number(trimmed) + if (trimmed !== '' && Number.isFinite(asNumber) && String(asNumber) === trimmed) { + return asNumber + } + return trimmed + } + const asNumber = Number(value) + if (Number.isFinite(asNumber)) { + return asNumber + } + return value +} + +const parseMicroserviceImages = async (fileImages, options = {}) => { + const { templateMode = false } = options // Could be undefined if patch call if (!fileImages) { return { registryId: undefined, images: undefined, catalogItemId: undefined } } - if (fileImages.catalogId > 0) { - return { registryId: undefined, images: undefined, catalogItemId: fileImages.catalogId } - } - const registryByName = { - remote: 1, - local: 2 + if (fileImages.catalogId != null && fileImages.catalogId !== '') { + const catalogItemId = asIdOrPlaceholder(fileImages.catalogId) + if (templateMode && typeof catalogItemId === 'string') { + return { registryId: undefined, images: undefined, catalogItemId } + } + if (typeof catalogItemId === 'number' && catalogItemId > 0) { + return { registryId: undefined, images: undefined, catalogItemId } + } } const images = mapImages(fileImages) - const registryId = fileImages.registry != null ? registryByName[fileImages.registry] || Number(fileImages.registry) : 1 + let registryId + if (fileImages.registry != null && fileImages.registry !== '') { + registryId = asIdOrPlaceholder(fileImages.registry) + } else if (!templateMode) { + registryId = 1 + } return { registryId, catalogItemId: undefined, images } } +function parseEnvVariables (envArray) { + if (!envArray || !Array.isArray(envArray)) { + return [] + } + + return envArray.map(env => { + if (!env || typeof env !== 'object') { + throw new Errors.ValidationError('Invalid environment variable format') + } + + if (!env.key) { + throw new Errors.ValidationError('Environment variable must have a key') + } + + const envVar = { + key: env.key.toString() + } + + const hasValue = Object.hasOwn(env, 'value') + const hasValueFromSecret = Object.hasOwn(env, 'valueFromSecret') + const hasValueFromConfigMap = Object.hasOwn(env, 'valueFromConfigMap') + + const valueCount = [hasValue, hasValueFromSecret, hasValueFromConfigMap].filter(Boolean).length + + if (valueCount === 0) { + throw new Errors.ValidationError(`Environment variable '${env.key}' must have either value, valueFromSecret, or valueFromConfigMap`) + } + + if (valueCount > 1) { + throw new Errors.ValidationError(`Environment variable '${env.key}' can only have one of: value, valueFromSecret, or valueFromConfigMap`) + } + + if (hasValue) { + envVar.value = env.value.toString() + } + + if (hasValueFromSecret) { + if (typeof env.valueFromSecret !== 'string') { + throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be a string`) + } + const parts = env.valueFromSecret.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be in format 'secret-name/key'`) + } + envVar.valueFromSecret = env.valueFromSecret + } + + if (hasValueFromConfigMap) { + if (typeof env.valueFromConfigMap !== 'string') { + throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be a string`) + } + const parts = env.valueFromConfigMap.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be in format 'configmap-name/key'`) + } + envVar.valueFromConfigMap = env.valueFromConfigMap + } + + return envVar + }) +} + function parseMicroserviceTemplateRef (template) { if (template == null) { return undefined @@ -275,73 +382,6 @@ const parseMicroserviceYAML = async (microservice) => { const { registryId, catalogItemId, images } = await parseMicroserviceImages(microservice.images) const container = microservice.container || {} - // Parse environment variables with support for value, valueFromSecret, and valueFromConfigMap - const parseEnvVariables = (envArray) => { - if (!envArray || !Array.isArray(envArray)) { - return [] - } - - return envArray.map(env => { - if (!env || typeof env !== 'object') { - throw new Errors.ValidationError('Invalid environment variable format') - } - - if (!env.key) { - throw new Errors.ValidationError('Environment variable must have a key') - } - - const envVar = { - key: env.key.toString() - } - - // Check that exactly one of value, valueFromSecret, or valueFromConfigMap is provided - const hasValue = Object.hasOwn(env, 'value') - const hasValueFromSecret = Object.hasOwn(env, 'valueFromSecret') - const hasValueFromConfigMap = Object.hasOwn(env, 'valueFromConfigMap') - - const valueCount = [hasValue, hasValueFromSecret, hasValueFromConfigMap].filter(Boolean).length - - if (valueCount === 0) { - throw new Errors.ValidationError(`Environment variable '${env.key}' must have either value, valueFromSecret, or valueFromConfigMap`) - } - - if (valueCount > 1) { - throw new Errors.ValidationError(`Environment variable '${env.key}' can only have one of: value, valueFromSecret, or valueFromConfigMap`) - } - - // Handle simple value - if (hasValue) { - envVar.value = env.value.toString() - } - - // Handle valueFromSecret - if (hasValueFromSecret) { - if (typeof env.valueFromSecret !== 'string') { - throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be a string`) - } - const parts = env.valueFromSecret.split('/') - if (parts.length !== 2 || !parts[0] || !parts[1]) { - throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be in format 'secret-name/key'`) - } - envVar.valueFromSecret = env.valueFromSecret - } - - // Handle valueFromConfigMap - if (hasValueFromConfigMap) { - if (typeof env.valueFromConfigMap !== 'string') { - throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be a string`) - } - const parts = env.valueFromConfigMap.split('/') - if (parts.length !== 2 || !parts[0] || !parts[1]) { - throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be in format 'configmap-name/key'`) - } - envVar.valueFromConfigMap = env.valueFromConfigMap - } - - return envVar - }) - } - const commands = container.commands !== undefined ? container.commands : container.cmd const sysctls = container.sysctls !== undefined ? CatalogContainer.normalizeSysctls(container.sysctls) @@ -788,6 +828,96 @@ async function parseRuntimeClassFile (fileContent, options = {}) { } } +function setContainerFieldIfPresent (data, container, dataKey, containerKey, transform) { + if (!Object.hasOwn(container, containerKey)) { + return + } + data[dataKey] = transform ? transform(container[containerKey]) : container[containerKey] +} + +async function parseMicroserviceTemplateYAML (microservice) { + const { registryId, catalogItemId, images } = await parseMicroserviceImages(microservice.images, { templateMode: true }) + const container = microservice.container || {} + const microserviceData = {} + + if (microservice.config != null) { + microserviceData.config = JSON.stringify(microservice.config) + } + if (microservice.application !== undefined) { + microserviceData.application = microservice.application + } + if (lget(microservice, 'agent.name') !== undefined) { + microserviceData.agentName = lget(microservice, 'agent.name') + } + if (catalogItemId !== undefined) { + microserviceData.catalogItemId = catalogItemId + } + if (registryId !== undefined) { + microserviceData.registryId = registryId + } + if (images && images.length > 0) { + microserviceData.images = images + } + if (microservice.models !== undefined) { + microserviceData.models = microservice.models + } + if (microservice.schedule !== undefined) { + microserviceData.schedule = microservice.schedule + } + if (microservice.serviceAccount !== undefined) { + microserviceData.serviceAccount = microservice.serviceAccount + } + if (microservice.natsEnabled !== undefined) { + microserviceData.natsEnabled = microservice.natsEnabled + } + if (microservice.natsConfig !== undefined) { + microserviceData.natsConfig = microservice.natsConfig + } + + setContainerFieldIfPresent(microserviceData, container, 'hostNetworkMode', 'hostNetworkMode') + setContainerFieldIfPresent(microserviceData, container, 'isPrivileged', 'isPrivileged') + setContainerFieldIfPresent(microserviceData, container, 'pidMode', 'pidMode') + setContainerFieldIfPresent(microserviceData, container, 'ipcMode', 'ipcMode') + setContainerFieldIfPresent(microserviceData, container, 'cpuSetCpus', 'cpuSetCpus') + setContainerFieldIfPresent(microserviceData, container, 'memoryLimit', 'memoryLimit') + setContainerFieldIfPresent(microserviceData, container, 'runAsUser', 'runAsUser') + setContainerFieldIfPresent(microserviceData, container, 'runAsGroup', 'runAsGroup') + setContainerFieldIfPresent(microserviceData, container, 'readOnlyRootFilesystem', 'readOnlyRootFilesystem') + setContainerFieldIfPresent(microserviceData, container, 'platform', 'platform') + setContainerFieldIfPresent(microserviceData, container, 'runtime', 'runtime') + setContainerFieldIfPresent(microserviceData, container, 'cdiDevices', 'cdiDevices') + setContainerFieldIfPresent(microserviceData, container, 'cpus', 'cpus') + setContainerFieldIfPresent(microserviceData, container, 'memoryReservation', 'memoryReservation') + setContainerFieldIfPresent(microserviceData, container, 'memorySwap', 'memorySwap') + setContainerFieldIfPresent(microserviceData, container, 'shmSize', 'shmSize') + setContainerFieldIfPresent(microserviceData, container, 'workingDir', 'workingDir') + setContainerFieldIfPresent(microserviceData, container, 'entrypoint', 'entrypoint') + setContainerFieldIfPresent(microserviceData, container, 'ulimits', 'ulimits') + setContainerFieldIfPresent(microserviceData, container, 'devices', 'devices') + setContainerFieldIfPresent(microserviceData, container, 'tmpfs', 'tmpfs') + setContainerFieldIfPresent(microserviceData, container, 'capAdd', 'capAdd') + setContainerFieldIfPresent(microserviceData, container, 'capDrop', 'capDrop') + setContainerFieldIfPresent(microserviceData, container, 'ports', 'ports') + setContainerFieldIfPresent(microserviceData, container, 'volumes', 'volumeMappings') + setContainerFieldIfPresent(microserviceData, container, 'extraHosts', 'extraHosts') + setContainerFieldIfPresent(microserviceData, container, 'healthCheck', 'healthCheck') + if (container.annotations != null) { + microserviceData.annotations = JSON.stringify(container.annotations) + } + if (container.sysctls !== undefined) { + microserviceData.sysctls = CatalogContainer.normalizeSysctls(container.sysctls) + } + if (Object.hasOwn(container, 'commands') || Object.hasOwn(container, 'cmd')) { + microserviceData.commands = container.commands !== undefined ? container.commands : container.cmd + } + if (Object.hasOwn(container, 'env')) { + microserviceData.env = parseEnvVariables(container.env) + } + + _deleteUndefinedFields(microserviceData) + return microserviceData +} + async function parseMicroserviceTemplateFile (fileContent, options = {}) { try { const doc = yaml.load(fileContent) @@ -817,12 +947,11 @@ async function parseMicroserviceTemplateFile (fileContent, options = {}) { throw new Errors.ValidationError('MicroserviceTemplate spec.microservice is required') } - const microservice = await parseMicroserviceYAML(spec.microservice) + const microservice = await parseMicroserviceTemplateYAML(spec.microservice) delete microservice.name - delete microservice.application delete microservice.iofogUuid - delete microservice.agentName delete microservice.flowId + delete microservice.template const result = { name: lget(doc, 'metadata.name', undefined), diff --git a/test/src/services/yaml-parser-service.test.js b/test/src/services/yaml-parser-service.test.js index 086435b8..dcfde491 100644 --- a/test/src/services/yaml-parser-service.test.js +++ b/test/src/services/yaml-parser-service.test.js @@ -512,7 +512,74 @@ spec: expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) expect(result.microservice.env).to.eql([{ key: 'PORT', value: '{{ port }}' }]) expect(result.microservice).to.not.have.property('name') - expect(result.microservice).to.not.have.property('application') + expect(result.microservice).to.not.have.property('pidMode') + expect(result.microservice).to.not.have.property('healthCheck') + }) + + it('preserves template placeholders and typed variable defaults', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: MicroserviceTemplate +metadata: + name: ms-template2 +spec: + description: Template for creating microservices + variables: + - key: application + defaultValue: test-app + - key: agent-name + defaultValue: lima + - key: nats-access + defaultValue: true + - key: registry-id + defaultValue: 5 + - key: schedule + defaultValue: 50 + - key: shm-size + defaultValue: 1024 + microservice: + application: "{{application}}" + schedule: "{{schedule}}" + agent: + name: "{{agent-name}}" + natsConfig: + natsAccess: "{{nats-access}}" + natsRule: "{{nats-rule}}" + images: + registry: "{{registry-id}}" + arm64: "{{arm64-image}}" + amd64: "{{amd64-image}}" + models: + bindPath: "{{bind-path}}" + permissions: "{{permissions}}" + items: + - name: "{{model1}}" + container: + hostNetworkMode: false + shmSize: "{{shm-size}}" + commands: [] +` + const result = await YamlParserService.parseMicroserviceTemplateFile(yaml) + expect(result.variables).to.eql([ + { key: 'application', defaultValue: 'test-app' }, + { key: 'agent-name', defaultValue: 'lima' }, + { key: 'nats-access', defaultValue: true }, + { key: 'registry-id', defaultValue: 5 }, + { key: 'schedule', defaultValue: 50 }, + { key: 'shm-size', defaultValue: 1024 } + ]) + expect(result.microservice.application).to.equal('{{application}}') + expect(result.microservice.agentName).to.equal('{{agent-name}}') + expect(result.microservice.registryId).to.equal('{{registry-id}}') + expect(result.microservice.schedule).to.equal('{{schedule}}') + expect(result.microservice.shmSize).to.equal('{{shm-size}}') + expect(result.microservice.natsConfig).to.eql({ + natsAccess: '{{nats-access}}', + natsRule: '{{nats-rule}}' + }) + expect(result.microservice.models.items).to.eql([{ name: '{{model1}}' }]) + expect(result.microservice).to.not.have.property('pidMode') + expect(result.microservice).to.not.have.property('cmd') }) it('accepts iofog.org/v3 apiVersion', async () => { From dc6165e9c5091e83bbc2fe5e60420af5ddaede8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Thu, 17 Sep 2026 16:07:58 +0300 Subject: [PATCH 14/55] Keep application and agent placement in microservice templates and substitute typed defaults on deploy. --- src/routes/microserviceTemplates.js | 2 - src/services/microservice-template-service.js | 15 ++-- .../microservice-template-service.test.js | 87 +++++++++++++++++-- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/routes/microserviceTemplates.js b/src/routes/microserviceTemplates.js index 2822478b..1a97d6b0 100644 --- a/src/routes/microserviceTemplates.js +++ b/src/routes/microserviceTemplates.js @@ -68,7 +68,6 @@ module.exports = [ { method: 'patch', path: '/api/v3/microserviceTemplates/:name', - supportSubstitution: true, middleware: async (req, res) => { logger.apiReq(req) await rbacMiddleware.protect()(req, res, async () => { @@ -96,7 +95,6 @@ module.exports = [ { method: 'post', path: '/api/v3/microserviceTemplates', - supportSubstitution: true, middleware: async (req, res) => { logger.apiReq(req) await rbacMiddleware.protect()(req, res, async () => { diff --git a/src/services/microservice-template-service.js b/src/services/microservice-template-service.js index de69d997..56359f68 100644 --- a/src/services/microservice-template-service.js +++ b/src/services/microservice-template-service.js @@ -6,7 +6,7 @@ const MicroserviceTemplateVariableManager = require('../data/managers/microservi const TransactionDecorator = require('../decorators/transaction-decorator') const Validator = require('../schemas') -const IDENTITY_FIELDS = ['name', 'application', 'iofogUuid', 'agentName', 'flowId', 'template'] +const INSTANCE_FIELDS = ['name', 'iofogUuid', 'flowId', 'template'] function _toPlain (row) { if (!row) { @@ -43,9 +43,9 @@ function _parseStoredDefaultValue (defaultValue) { } } -function stripIdentityFields (microservice) { +function stripInstanceFields (microservice) { const clone = { ...(microservice || {}) } - for (const field of IDENTITY_FIELDS) { + for (const field of INSTANCE_FIELDS) { delete clone[field] } return clone @@ -172,7 +172,7 @@ async function createMicroserviceTemplateEndpoint (data, transaction) { const created = await MicroserviceTemplateManager.create({ name: data.name, description: data.description != null ? data.description : '', - microserviceJSON: JSON.stringify(stripIdentityFields(data.microservice)) + microserviceJSON: JSON.stringify(stripInstanceFields(data.microservice)) }, transaction) try { @@ -194,7 +194,7 @@ async function updateMicroserviceTemplateEndpoint (name, data, transaction) { const current = toWireTemplate(existing) const nextMicroservice = data.microservice !== undefined - ? stripIdentityFields(data.microservice) + ? stripInstanceFields(data.microservice) : current.microservice const nextDescription = data.description !== undefined ? data.description : current.description @@ -237,7 +237,7 @@ async function getMicroserviceDataFromTemplate (deploymentData, isCLI, transacti throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_INVALID, deploymentData.name)) } - const spec = stripIdentityFields(template.microservice) + const spec = stripInstanceFields(template.microservice) const parsedConfig = _parseJsonObjectField(spec.config) if (parsedConfig !== undefined) { spec.config = parsedConfig @@ -274,5 +274,6 @@ module.exports = { deleteMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(deleteMicroserviceTemplateEndpoint), getMicroserviceDataFromTemplate, toWireTemplate, - stripIdentityFields + stripInstanceFields, + stripIdentityFields: stripInstanceFields } diff --git a/test/src/services/microservice-template-service.test.js b/test/src/services/microservice-template-service.test.js index 51fa9a15..1250e345 100644 --- a/test/src/services/microservice-template-service.test.js +++ b/test/src/services/microservice-template-service.test.js @@ -51,7 +51,7 @@ describe('Microservice Template Service', () => { $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(stubTemplateRow()) }) - it('creates a template and strips instance identity fields', async () => { + it('creates a template and strips deploy instance fields only', async () => { const result = await MicroserviceTemplateService.createMicroserviceTemplateEndpoint({ name: 'nginx-edge', description: 'nginx at the edge', @@ -59,16 +59,19 @@ describe('Microservice Template Service', () => { microservice: { ...templateSpec(), name: 'should-not-store', - application: 'should-not-store', + application: '{{ app-name }}', + agentName: '{{ agent-name }}', iofogUuid: 'fog-uuid' } }, transaction) expect(result.name).to.equal('nginx-edge') - expect(result.microservice).to.not.have.property('name') - expect(result.microservice).to.not.have.property('application') - expect(result.microservice).to.not.have.property('iofogUuid') - expect(result.microservice.models.items).to.eql([{ name: 'test-model' }]) + const storedMicroservice = JSON.parse(MicroserviceTemplateManager.create.getCall(0).args[0].microserviceJSON) + expect(storedMicroservice).to.not.have.property('name') + expect(storedMicroservice.application).to.equal('{{ app-name }}') + expect(storedMicroservice.agentName).to.equal('{{ agent-name }}') + expect(storedMicroservice).to.not.have.property('iofogUuid') + expect(storedMicroservice.models.items).to.eql([{ name: 'test-model' }]) expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) expect(result.variables[0].key).to.equal('port') expect(result.variables[0].defaultValue).to.equal('80') @@ -211,6 +214,78 @@ describe('Microservice Template Service', () => { expect(result.env).to.eql([{ key: 'PORT', value: '80' }]) }) + it('substitutes typed default values into the microservice spec', async () => { + MicroserviceTemplateManager.findOnePopulated.resolves(stubTemplateRow({ + microserviceJSON: JSON.stringify({ + application: '{{application}}', + agentName: '{{agent-name}}', + registryId: '{{registry-id}}', + schedule: '{{schedule}}', + shmSize: '{{shm-size}}', + natsConfig: { + natsAccess: '{{nats-access}}', + natsRule: '{{nats-rule}}' + }, + models: { + bindPath: '{{bind-path}}', + permissions: '{{permissions}}', + items: [ + { name: '{{model1}}' }, + { name: '{{model2}}' }, + { name: '{{model3}}' } + ] + }, + images: [ + { archId: 1, containerImage: '{{amd64-image}}' }, + { archId: 2, containerImage: '{{arm64-image}}' } + ] + }), + variables: [ + { key: 'application', defaultValue: JSON.stringify('test-app') }, + { key: 'agent-name', defaultValue: JSON.stringify('lima') }, + { key: 'nats-access', defaultValue: JSON.stringify(true) }, + { key: 'nats-rule', defaultValue: JSON.stringify('default-user') }, + { key: 'schedule', defaultValue: JSON.stringify(50) }, + { key: 'bind-path', defaultValue: JSON.stringify('/models') }, + { key: 'permissions', defaultValue: JSON.stringify('ro') }, + { key: 'model1', defaultValue: JSON.stringify('model1') }, + { key: 'model2', defaultValue: JSON.stringify('model2') }, + { key: 'model3', defaultValue: JSON.stringify('') }, + { key: 'registry-id', defaultValue: JSON.stringify(5) }, + { key: 'arm64-image', defaultValue: JSON.stringify('dhi.io/clickhouse-server:26.7') }, + { key: 'amd64-image', defaultValue: JSON.stringify('dhi.io/clickhouse-server:26.7') }, + { key: 'shm-size', defaultValue: JSON.stringify(1024) } + ] + })) + + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge' + }, false, transaction) + + expect(result.application).to.equal('test-app') + expect(result.agentName).to.equal('lima') + expect(result.registryId).to.equal(5) + expect(result.schedule).to.equal(50) + expect(result.shmSize).to.equal(1024) + expect(result.natsConfig).to.eql({ + natsAccess: true, + natsRule: 'default-user' + }) + expect(result.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [ + { name: 'model1' }, + { name: 'model2' }, + { name: '' } + ] + }) + expect(result.images).to.eql([ + { archId: 1, containerImage: 'dhi.io/clickhouse-server:26.7' }, + { archId: 2, containerImage: 'dhi.io/clickhouse-server:26.7' } + ]) + }) + it('accepts application-style array variables', async () => { const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ name: 'nginx-edge', From 8f3fae696e4b315340529d83385c5fadf2dafaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Thu, 17 Sep 2026 16:08:05 +0300 Subject: [PATCH 15/55] Parse application template variable defaults without failing on missing or non-JSON values. --- src/services/application-template-service.js | 41 +++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/services/application-template-service.js b/src/services/application-template-service.js index 7b1e14b8..6328dd1e 100644 --- a/src/services/application-template-service.js +++ b/src/services/application-template-service.js @@ -11,6 +11,32 @@ const ApplicationTemplateVariableManager = require('../data/managers/application const TransactionDecorator = require('../decorators/transaction-decorator') const Validator = require('../schemas') +function _parseStoredDefaultValue (defaultValue) { + if (defaultValue === undefined || defaultValue === null) { + return undefined + } + if (typeof defaultValue !== 'string') { + return defaultValue + } + try { + return JSON.parse(defaultValue) + } catch (error) { + return defaultValue + } +} + +function _buildDefaultVariableMap (variables) { + return (variables || []).reduce((acc, variable) => { + if (variable && variable.key != null) { + const parsed = _parseStoredDefaultValue(variable.defaultValue) + if (parsed !== undefined) { + acc[variable.key] = parsed + } + } + return acc + }, {}) +} + const createApplicationTemplateEndPoint = async function (applicationTemplateData, isCLI, transaction) { // Add a name field to pass schema validation using the applicationCreate schema applicationTemplateData.application = { ...applicationTemplateData.application, name: 'validation' } @@ -157,9 +183,16 @@ const _buildGetApplicationObj = function (applicationDBObj) { const JSONData = applicationDBObj.toJSON() JSONData.application = JSON.parse(JSONData.applicationJSON || null) JSONData.variables = (JSONData.variables || []).map(v => { - if (v.defaultValue === undefined) { return v } + if (v.defaultValue === undefined || v.defaultValue === null) { + return v + } + const parsed = _parseStoredDefaultValue(v.defaultValue) + if (parsed === undefined) { + return v + } return { - ...v, defaultValue: JSON.parse(v.defaultValue) + ...v, + defaultValue: parsed } }) delete JSONData.applicationJSON @@ -208,9 +241,7 @@ const getApplicationDataFromTemplate = async function (deploymentData, isCLI, tr const newApplication = applicationTemplate.application // Replace variables - const defaultVariablesValues = (applicationTemplate.variables || []).reduce((acc, v) => { - return { ...acc, [v.key]: JSON.parse(v.defaultValue) } - }, {}) + const defaultVariablesValues = _buildDefaultVariableMap(applicationTemplate.variables) const userProvidedVariables = (deploymentData.variables || []).reduce((acc, v) => { return { ...acc, [v.key]: v.value } }, {}) From da3d0b019750941a551ded2c4776975f134689b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Thu, 17 Sep 2026 16:08:10 +0300 Subject: [PATCH 16/55] Name the application template YAML upload field template. --- docs/swagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 586eadb6..595e2102 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -923,7 +923,7 @@ paths: schema: type: object properties: - application: + template: type: string format: binary responses: From d351f9232ac4e9715359a316bd4ceaf01c528826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Thu, 17 Sep 2026 16:24:05 +0300 Subject: [PATCH 17/55] Bump the embedded EdgeOps Console default to v1.1.0-rc.1. --- .env.example | 2 +- .github/actions/set-build-env/action.yml | 2 +- CHANGELOG.md | 1 + Dockerfile | 4 ++-- Makefile | 4 ++-- scripts/build-console-dev.js | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 1fd43f30..e401c5b5 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.0.13 +EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index b36ee5cb..e79ed4d2 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.0.13"; fi + if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.1"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index b90ab6b9..e8b9d556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Fleet models, RuntimeClass, registry extensions, microservice catalog PATCH, Mic - **MS deploy** — auto-links referenced fleet models and RuntimeClass on edgelet agents; validates runtime against `availableRuntimes`. - **RBAC + swagger** — models, runtimeClasses, microserviceTemplates, and agent model/runtime routes documented and catalogued. +- Embedded **EdgeOps Console** default version **v1.0.13** → **v1.1.0-rc.1** ### Removed diff --git a/Dockerfile b/Dockerfile index 818bb46b..8fcc3ecf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM node:24-bookworm@sha256:6dac556d980b7f0e5498d08f08cee0ca67798b4ad6c23964a9214920e67758d0 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.0.13 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -50,7 +50,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:0838eaa65048d6c76749b92f423ee73af7fdd72b32b365a0fdc200b3ab74f363 -ARG EDGEOPS_CONSOLE_VERSION=v1.0.13 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index b68149b8..9e323678 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.13 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.0.13 +EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.1 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index e9ee9359..58989d29 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.13' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.1' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { From ea20843a615bb577a8b61bbcb2a7975ef10549b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 01:34:46 +0300 Subject: [PATCH 18/55] Fix iofog PATCH after upgrade when FogPlatformSpecs retains removed HAL fields. Whitelist platform spec fields on merge and upsert so stale bluetoothEnabled and abstractedHardwareEnabled keys from pre-HAL-removal rows no longer block PATCH; improve validation errors to name the rejected field. --- CHANGELOG.md | 8 +++- .../managers/fog-platform-spec-manager.js | 8 ++-- src/schemas/fog-platform-spec.js | 40 ++++++++++++++----- test/src/data/fog-platform-managers.test.js | 23 ++++++++++- 4 files changed, 63 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8b9d556..08c151dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,13 @@ # Changelog +## [v3.9.0-rc.2] -## [Unreleased] +### Fixed + +- **`PATCH /api/v3/iofog/{uuid}` after Controller upgrade** — stale `bluetoothEnabled` / `abstractedHardwareEnabled` keys in `"FogPlatformSpecs".spec_json` (written before HAL removal) caused validation failure with misleading `Invalid fog platform spec field 'instance'`. Platform spec merge and upsert now whitelist allowed fields only; validation errors name the rejected field. + + +## [v3.9.0-rc.1] Fleet models, RuntimeClass, registry extensions, microservice catalog PATCH, MicroserviceTemplate, and HAL/BLE removal for Edgelet v1.1+ handoff. Targets **v3.9.0** schema (user tags). diff --git a/src/data/managers/fog-platform-spec-manager.js b/src/data/managers/fog-platform-spec-manager.js index 492fe1c4..28d231b6 100644 --- a/src/data/managers/fog-platform-spec-manager.js +++ b/src/data/managers/fog-platform-spec-manager.js @@ -3,7 +3,8 @@ const models = require('../models') const { validateFogPlatformSpec, parseSpecJson, - serializeSpecJson + serializeSpecJson, + pickPlatformSpecFields } = require('../../schemas/fog-platform-spec') class FogPlatformSpecManager extends BaseManager { @@ -24,8 +25,9 @@ class FogPlatformSpecManager extends BaseManager { } async upsertSpec (fogUuid, specObject, transaction) { - await validateFogPlatformSpec(specObject) - const specJson = serializeSpecJson(specObject) + const sanitized = pickPlatformSpecFields(specObject) + await validateFogPlatformSpec(sanitized) + const specJson = serializeSpecJson(sanitized) const existing = await this.findOne({ fogUuid }, transaction) if (existing) { diff --git a/src/schemas/fog-platform-spec.js b/src/schemas/fog-platform-spec.js index 6d3309c0..30b038f3 100644 --- a/src/schemas/fog-platform-spec.js +++ b/src/schemas/fog-platform-spec.js @@ -93,14 +93,23 @@ function serializeJson (object, label) { return json } +function formatFogPlatformSpecValidationError (error) { + let property = (error.property || '').replace(/^instance\.?/, '') + if (!property || property === 'instance') { + const match = error.message && error.message.match(/additional property "([^"]+)"/) + if (match) { + property = match[1] + } + } + return property + ? `Invalid fog platform spec field '${property}'` + : 'Invalid fog platform spec' +} + async function validateFogPlatformSpec (object) { const response = validator.validate(object || {}, fogPlatformSpec) if (!response.valid) { - const error = response.errors[0] - const property = (error.property || '').replace('instance.', '') - throw new Errors.ValidationError( - property ? `Invalid fog platform spec field '${property}'` : 'Invalid fog platform spec' - ) + throw new Errors.ValidationError(formatFogPlatformSpecValidationError(response.errors[0])) } return object } @@ -173,18 +182,26 @@ function normalizeSpecTags (tags) { return tags.map((tag) => (typeof tag === 'string' ? { value: tag } : tag)) } -function buildPlatformSpecFromFogData (fogData, options = {}) { +function pickPlatformSpecFields (source) { + if (!source) { + return {} + } const spec = {} for (const field of FOG_PLATFORM_SPEC_SCALAR_FIELDS) { - if (fogData[field] !== undefined) { - spec[field] = fogData[field] + if (source[field] !== undefined) { + spec[field] = source[field] } } for (const field of FOG_PLATFORM_SPEC_ARRAY_FIELDS) { - if (fogData[field] !== undefined) { - spec[field] = field === 'tags' ? normalizeSpecTags(fogData[field]) : fogData[field] + if (source[field] !== undefined) { + spec[field] = field === 'tags' ? normalizeSpecTags(source[field]) : source[field] } } + return spec +} + +function buildPlatformSpecFromFogData (fogData, options = {}) { + const spec = pickPlatformSpecFields(fogData) if (options.applyCreateDefaults) { if (spec.routerMode === undefined) { spec.routerMode = 'edge' @@ -197,7 +214,7 @@ function buildPlatformSpecFromFogData (fogData, options = {}) { } function mergePlatformSpecPatch (existingSpec, patchFogData) { - const merged = { ...(existingSpec || {}) } + const merged = pickPlatformSpecFields(existingSpec) for (const field of FOG_PLATFORM_SPEC_SCALAR_FIELDS) { if (patchFogData[field] !== undefined) { merged[field] = patchFogData[field] @@ -228,6 +245,7 @@ module.exports = { serializeConditionsJson, FOG_PLATFORM_SPEC_SCALAR_FIELDS, FOG_PLATFORM_SPEC_ARRAY_FIELDS, + pickPlatformSpecFields, buildPlatformSpecFromFogData, mergePlatformSpecPatch } diff --git a/test/src/data/fog-platform-managers.test.js b/test/src/data/fog-platform-managers.test.js index f7e8e4fd..a159e19d 100644 --- a/test/src/data/fog-platform-managers.test.js +++ b/test/src/data/fog-platform-managers.test.js @@ -7,7 +7,8 @@ const { parseSpecJson, serializeSpecJson, parseSpecSnapshot, - serializeSpecSnapshot + serializeSpecSnapshot, + mergePlatformSpecPatch } = require('../../../src/schemas/fog-platform-spec') const FogPlatformReconcileTaskManager = require('../../../src/data/managers/fog-platform-reconcile-task-manager') const ServicePlatformReconcileTaskManager = require('../../../src/data/managers/service-platform-reconcile-task-manager') @@ -28,9 +29,29 @@ describe('Fog platform spec schema', () => { throw new Error('expected validation to fail') } catch (error) { expect(error.name).to.equal('ValidationError') + expect(error.message).to.equal("Invalid fog platform spec field 'unknownField'") } }) + it('mergePlatformSpecPatch drops removed fields from existing spec', async () => { + const merged = mergePlatformSpecPatch({ + routerMode: 'edge', + natsMode: 'leaf', + host: '1.2.3.4', + bluetoothEnabled: false, + abstractedHardwareEnabled: false + }, { + host: '5.6.7.8' + }) + + expect(merged).to.eql({ + routerMode: 'edge', + natsMode: 'leaf', + host: '5.6.7.8' + }) + await validateFogPlatformSpec(merged) + }) + it('round-trips spec JSON through parse and serialize', () => { const spec = { routerMode: 'interior', From 425c2f3c9875a31f9c0548fdfa48159feb764c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 01:53:15 +0300 Subject: [PATCH 19/55] Fix RuntimeClass link to require fog availableRuntimes. Reject POST /runtimeClasses/:name/link when the class name is not in the target fog's availableRuntimes, matching microservice deploy validation. --- CHANGELOG.md | 1 + docs/swagger.yaml | 2 +- src/helpers/error-messages.js | 1 + src/services/runtime-class-service.js | 30 +++++++++++++++++++ .../fleet-models-runtime-sqlite.test.js | 10 +++++++ .../services/runtime-class-service.test.js | 11 ++++++- 6 files changed, 53 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08c151dd..890aad79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - **`PATCH /api/v3/iofog/{uuid}` after Controller upgrade** — stale `bluetoothEnabled` / `abstractedHardwareEnabled` keys in `"FogPlatformSpecs".spec_json` (written before HAL removal) caused validation failure with misleading `Invalid fog platform spec field 'instance'`. Platform spec merge and upsert now whitelist allowed fields only; validation errors name the rejected field. +- **`POST /api/v3/runtimeClasses/{name}/link`** — refused link when the RuntimeClass name is not in the target fog `availableRuntimes` (same rule as microservice deploy runtime validation). ## [v3.9.0-rc.1] diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 595e2102..944ac471 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -6098,7 +6098,7 @@ paths: tags: - RuntimeClasses summary: Links a RuntimeClass to fog nodes - description: Attach is refused when fog containerEngine is not edgelet. + description: Attach is refused when fog containerEngine is not edgelet or when the RuntimeClass name is not in the fog availableRuntimes list. operationId: linkRuntimeClass parameters: - in: path diff --git a/src/helpers/error-messages.js b/src/helpers/error-messages.js index 2d629bfb..768bb7cb 100644 --- a/src/helpers/error-messages.js +++ b/src/helpers/error-messages.js @@ -148,6 +148,7 @@ module.exports = { RUNTIME_CLASS_IN_USE: 'Cannot delete RuntimeClass \'{}\': pinned by microservices {}', RUNTIME_CLASS_UNLINK_IN_USE: 'Cannot unlink RuntimeClass \'{}\' from agent {}: pinned by microservices {}', RUNTIME_CLASS_LINK_REQUIRES_EDGELET: 'Cannot link RuntimeClass to agent {}: container engine must be edgelet', + RUNTIME_CLASS_LINK_NOT_AVAILABLE: 'Cannot link RuntimeClass \'{}\' to agent \'{}\': runtime is not in availableRuntimes', RUNTIME_CLASS_MISSING: 'RuntimeClass \'{}\' does not exist', MICROSERVICE_CATALOG_BIND_PATH_REQUIRED: 'Model catalog bindPath is required when items are present', MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE: 'Model catalog bindPath must be an absolute container path', diff --git a/src/services/runtime-class-service.js b/src/services/runtime-class-service.js index 2f5c6be3..274f5071 100644 --- a/src/services/runtime-class-service.js +++ b/src/services/runtime-class-service.js @@ -44,6 +44,35 @@ function _warnIfUnknownHandler (name, handler) { ) } +function _parseFogAvailableRuntimes (fog) { + if (!fog || fog.availableRuntimes == null || fog.availableRuntimes === '') { + return [] + } + if (Array.isArray(fog.availableRuntimes)) { + return fog.availableRuntimes + } + try { + const parsed = JSON.parse(fog.availableRuntimes) + return Array.isArray(parsed) ? parsed : [] + } catch (error) { + return [] + } +} + +function _assertRuntimeClassAvailableOnFog (runtimeClassName, agent) { + const availableRuntimes = _parseFogAvailableRuntimes(agent) + if (!availableRuntimes.includes(runtimeClassName)) { + const agentLabel = agent.name || agent.uuid || 'agent' + throw new Errors.ValidationError( + AppHelper.formatMessage( + ErrorMessages.RUNTIME_CLASS_LINK_NOT_AVAILABLE, + runtimeClassName, + agentLabel + ) + ) + } +} + async function findLinkedFogUuids (runtimeClass, transaction) { if (!runtimeClass || typeof runtimeClass.getFogs !== 'function') { return [] @@ -168,6 +197,7 @@ async function linkRuntimeClassEndpoint (name, fogUuids, transaction) { AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_LINK_REQUIRES_EDGELET, fogUuid) ) } + _assertRuntimeClassAvailableOnFog(name, agent) await runtimeClass.addFog(agent, { transaction }) } diff --git a/test/src/integration/fleet-models-runtime-sqlite.test.js b/test/src/integration/fleet-models-runtime-sqlite.test.js index 1a7a77f5..951a2fc2 100644 --- a/test/src/integration/fleet-models-runtime-sqlite.test.js +++ b/test/src/integration/fleet-models-runtime-sqlite.test.js @@ -151,6 +151,7 @@ describe('fleet models and runtime sqlite integration', function () { const { expect } = require('chai') const RuntimeClassService = require('../../../src/services/runtime-class-service') const AgentService = require('../../../src/services/agent-service') + const FogManager = require('../../../src/data/managers/iofog-manager') const suffix = uniqueSuffix() const edgeFog = await createEdgeFog(suffix) @@ -164,6 +165,15 @@ describe('fleet models and runtime sqlite integration', function () { 'integration.createRuntimeClass' ) + await runTx( + (transaction) => FogManager.update( + { uuid: edgeFog.uuid }, + { availableRuntimes: JSON.stringify([className]) }, + transaction + ), + 'integration.setAvailableRuntimesForLink' + ) + await runTx( (transaction) => RuntimeClassService.linkRuntimeClassEndpoint(className, [edgeFog.uuid], transaction), 'integration.linkRuntimeClass' diff --git a/test/src/services/runtime-class-service.test.js b/test/src/services/runtime-class-service.test.js index 3f9fa15b..4d99421c 100644 --- a/test/src/services/runtime-class-service.test.js +++ b/test/src/services/runtime-class-service.test.js @@ -147,7 +147,9 @@ describe('RuntimeClass Service', () => { runtimeClass = stubRuntimeClassRow() agent = { uuid: fogUuid, - containerEngine: 'edgelet' + name: 'edge-1', + containerEngine: 'edgelet', + availableRuntimes: ['spin'] } $sandbox.stub(RuntimeClassManager, 'findOne').resolves(runtimeClass) $sandbox.stub(FogManager, 'findOne').resolves(agent) @@ -174,6 +176,13 @@ describe('RuntimeClass Service', () => { expect(runtimeClass.addFog).to.not.have.been.called }) + it('refuses link when the runtime is not in availableRuntimes', async () => { + agent.availableRuntimes = ['runc'] + await expect(RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ValidationError, /availableRuntimes/) + expect(runtimeClass.addFog).to.not.have.been.called + }) + it('warns when the handler is not in the known catalog', async () => { runtimeClass.handler = 'custom-oci' await RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction) From 41e997cd0137a34762bf8fa3433edabe188b8e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 19:57:34 +0300 Subject: [PATCH 20/55] Add last-crash columns on microservice status. Persist lastError, lastErrorAt, and restartCount on MicroserviceStatuses for both fresh 3.9 installs and 3.8 upgrades. --- .../db_migration_mysql_baseline_v3.9.0.sql | 3 +++ .../mysql/db_migration_mysql_v3.9.0.sql | 3 +++ .../db_migration_pg_baseline_v3.9.0.sql | 3 +++ .../postgres/db_migration_pg_v3.9.0.sql | 3 +++ .../db_migration_sqlite_baseline_v3.9.0.sql | 3 +++ .../sqlite/db_migration_sqlite_v3.9.0.sql | 3 +++ src/data/models/microservicestatus.js | 18 ++++++++++++++++++ 7 files changed, 36 insertions(+) diff --git a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql index 512e1485..2c5ab65a 100644 --- a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql @@ -313,6 +313,9 @@ CREATE TABLE IF NOT EXISTS MicroserviceStatuses ( exec_session_ids TEXT, health_status TEXT, pod_id TEXT, + last_error TEXT, + last_error_at BIGINT DEFAULT 0, + restart_count INT DEFAULT 0, FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE ); diff --git a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql index 2d0b6c9c..6084f7d6 100644 --- a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql @@ -170,6 +170,9 @@ CREATE INDEX idx_microservice_model_items_microserviceUuid ON MicroserviceModelI CREATE INDEX idx_microservice_model_items_name ON MicroserviceModelItems (name); ALTER TABLE MicroserviceStatuses ADD COLUMN pod_id TEXT; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error TEXT; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error_at BIGINT DEFAULT 0; +ALTER TABLE MicroserviceStatuses ADD COLUMN restart_count INT DEFAULT 0; DROP TABLE IF EXISTS HWInfos; DROP TABLE IF EXISTS USBInfos; diff --git a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql index fde82076..545030e5 100644 --- a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql @@ -311,6 +311,9 @@ CREATE TABLE IF NOT EXISTS "MicroserviceStatuses" ( exec_session_ids TEXT, health_status TEXT, pod_id TEXT DEFAULT '', + last_error TEXT DEFAULT '', + last_error_at BIGINT DEFAULT 0, + restart_count INTEGER DEFAULT 0, FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE ); diff --git a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql index 0e2ab5e4..813b2bd2 100644 --- a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql @@ -167,6 +167,9 @@ CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON "Mic CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON "MicroserviceModelItems" (name); ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS pod_id TEXT DEFAULT ''; +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS last_error TEXT DEFAULT ''; +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS last_error_at BIGINT DEFAULT 0; +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS restart_count INTEGER DEFAULT 0; DROP TABLE IF EXISTS "HWInfos"; DROP TABLE IF EXISTS "USBInfos"; diff --git a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql index 6c741e22..f41ddc54 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql @@ -311,6 +311,9 @@ CREATE TABLE IF NOT EXISTS MicroserviceStatuses ( exec_session_ids TEXT, health_status TEXT, pod_id TEXT DEFAULT '', + last_error TEXT DEFAULT '', + last_error_at BIGINT DEFAULT 0, + restart_count INTEGER DEFAULT 0, FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE ); diff --git a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql index 164caf65..4eebde95 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql @@ -184,6 +184,9 @@ CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON Micr CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON MicroserviceModelItems (name); ALTER TABLE MicroserviceStatuses ADD COLUMN pod_id TEXT DEFAULT ''; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error TEXT DEFAULT ''; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error_at BIGINT DEFAULT 0; +ALTER TABLE MicroserviceStatuses ADD COLUMN restart_count INTEGER DEFAULT 0; DROP TABLE IF EXISTS HWInfos; DROP TABLE IF EXISTS USBInfos; diff --git a/src/data/models/microservicestatus.js b/src/data/models/microservicestatus.js index e6204803..5d179705 100644 --- a/src/data/models/microservicestatus.js +++ b/src/data/models/microservicestatus.js @@ -66,6 +66,24 @@ module.exports = (sequelize, DataTypes) => { defaultValue: '', field: 'error_message' }, + lastError: { + type: DataTypes.TEXT, + defaultValue: '', + field: 'last_error' + }, + lastErrorAt: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('lastErrorAt')) + }, + defaultValue: 0, + field: 'last_error_at' + }, + restartCount: { + type: DataTypes.INTEGER, + defaultValue: 0, + field: 'restart_count' + }, ipAddress: { type: DataTypes.TEXT, defaultValue: '', From ceb6edeb6b40c6892c94c5f8d318665c00d412bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 19:57:43 +0300 Subject: [PATCH 21/55] Normalize live meters and last-crash extras for microservice status. Zero CPU/memory unless status is RUNNING, and keep last-crash fields unless this report sends new ones. --- src/helpers/microservice-runtime-metrics.js | 131 +++++++++++ .../microservice-runtime-metrics.test.js | 205 ++++++++++++++++++ 2 files changed, 336 insertions(+) create mode 100644 src/helpers/microservice-runtime-metrics.js create mode 100644 test/src/helpers/microservice-runtime-metrics.test.js diff --git a/src/helpers/microservice-runtime-metrics.js b/src/helpers/microservice-runtime-metrics.js new file mode 100644 index 00000000..b0b80937 --- /dev/null +++ b/src/helpers/microservice-runtime-metrics.js @@ -0,0 +1,131 @@ +const LIVE_RUNTIME_STATUS = 'RUNNING' + +function isLiveRuntimeStatus (status) { + return status === LIVE_RUNTIME_STATUS +} + +function zeroRuntimeMetrics () { + return { + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + } +} + +function coerceRuntimeMetric (value) { + if (value === undefined || value === null || value === '') { + return 0 + } + const numeric = Number(value) + return Number.isFinite(numeric) ? numeric : 0 +} + +function applyRuntimeMetrics (patch, observedStatus) { + if (!patch) { + return patch + } + if (!isLiveRuntimeStatus(observedStatus)) { + Object.assign(patch, zeroRuntimeMetrics()) + } + return patch +} + +function _cloneStatusRow (statusRow) { + if (statusRow == null) { + return statusRow + } + if (typeof statusRow.get === 'function') { + return statusRow.get({ plain: true }) + } + if (statusRow.dataValues) { + return Object.assign({}, statusRow.dataValues) + } + return Object.assign({}, statusRow) +} + +function coerceCrashText (value) { + if (value === undefined || value === null) { + return undefined + } + if (typeof value === 'string') { + return value + } + if (typeof value === 'number' && Number.isFinite(value)) { + return String(value) + } + return undefined +} + +function coerceOptionalNonNegativeInt (value) { + if (value === undefined || value === null || value === '') { + return undefined + } + const numeric = Number(value) + if (!Number.isFinite(numeric) || numeric < 0) { + return undefined + } + return Math.floor(numeric) +} + +function applyCrashExtras (patch, reported) { + if (!patch || !reported) { + return patch + } + const lastErrorText = coerceCrashText(reported.lastError) + const lastErrorPresent = lastErrorText !== undefined && lastErrorText.length > 0 + const lastErrorAt = coerceOptionalNonNegativeInt(reported.lastErrorAt) + const lastErrorAtPresent = lastErrorAt !== undefined && lastErrorAt > 0 + const restartCountFieldSent = reported.restartCount !== undefined && + reported.restartCount !== null && reported.restartCount !== '' + const restartCount = coerceOptionalNonNegativeInt(reported.restartCount) + + if (lastErrorPresent) { + patch.lastError = lastErrorText + } + if (lastErrorAtPresent) { + patch.lastErrorAt = lastErrorAt + } + if (restartCountFieldSent) { + if (restartCount !== undefined) { + patch.restartCount = restartCount + } + } else if (lastErrorPresent || lastErrorAtPresent) { + patch.restartCount = 0 + } + return patch +} + +function projectCrashExtras (projected) { + if (!projected) { + return projected + } + if (projected.lastError == null) { + projected.lastError = '' + } + const lastErrorAt = Number(projected.lastErrorAt) + projected.lastErrorAt = Number.isFinite(lastErrorAt) ? lastErrorAt : 0 + const restartCount = Number(projected.restartCount) + projected.restartCount = Number.isFinite(restartCount) ? restartCount : 0 + return projected +} + +function projectStatusForApi (statusRow) { + const projected = _cloneStatusRow(statusRow) + if (!projected) { + return projected + } + applyRuntimeMetrics(projected, projected.status) + projectCrashExtras(projected) + return projected +} + +module.exports = { + isLiveRuntimeStatus, + zeroRuntimeMetrics, + coerceRuntimeMetric, + applyRuntimeMetrics, + applyCrashExtras, + projectCrashExtras, + projectStatusForApi +} diff --git a/test/src/helpers/microservice-runtime-metrics.test.js b/test/src/helpers/microservice-runtime-metrics.test.js new file mode 100644 index 00000000..b451bda0 --- /dev/null +++ b/test/src/helpers/microservice-runtime-metrics.test.js @@ -0,0 +1,205 @@ +const { expect } = require('chai') + +const { + isLiveRuntimeStatus, + zeroRuntimeMetrics, + coerceRuntimeMetric, + applyRuntimeMetrics, + applyCrashExtras, + projectStatusForApi +} = require('../../../src/helpers/microservice-runtime-metrics') + +describe('helpers/microservice-runtime-metrics', () => { + describe('isLiveRuntimeStatus()', () => { + it('is true only for RUNNING', () => { + expect(isLiveRuntimeStatus('RUNNING')).to.equal(true) + expect(isLiveRuntimeStatus('STOPPING')).to.equal(false) + expect(isLiveRuntimeStatus('STOPPED')).to.equal(false) + expect(isLiveRuntimeStatus('UNKNOWN')).to.equal(false) + }) + }) + + describe('zeroRuntimeMetrics()', () => { + it('returns zero cpu, memory, startTime, and operatingDuration', () => { + expect(zeroRuntimeMetrics()).to.eql({ + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }) + }) + }) + + describe('coerceRuntimeMetric()', () => { + it('treats omitted values as 0', () => { + expect(coerceRuntimeMetric(undefined)).to.equal(0) + expect(coerceRuntimeMetric(null)).to.equal(0) + expect(coerceRuntimeMetric('')).to.equal(0) + }) + + it('parses numeric strings', () => { + expect(coerceRuntimeMetric('12.5')).to.equal(12.5) + expect(coerceRuntimeMetric('8')).to.equal(8) + }) + + it('treats non-numeric values as 0', () => { + expect(coerceRuntimeMetric('n/a')).to.equal(0) + }) + }) + + describe('applyRuntimeMetrics()', () => { + it('leaves live RUNNING meters unchanged', () => { + const patch = { status: 'RUNNING', cpuUsage: 12.5, memoryUsage: 64, startTime: 9, operatingDuration: 10 } + expect(applyRuntimeMetrics(patch, 'RUNNING')).to.equal(patch) + expect(patch).to.include({ cpuUsage: 12.5, memoryUsage: 64, startTime: 9, operatingDuration: 10 }) + }) + + it('zeros meters for any non-RUNNING status', () => { + const patch = { + status: 'STOPPED', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10, + containerId: 'ctr-1', + errorMessage: 'exit 1', + podId: 'pod-1', + healthStatus: 'healthy' + } + applyRuntimeMetrics(patch, 'STOPPED') + expect(patch).to.include({ + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-1', + errorMessage: 'exit 1', + podId: 'pod-1', + healthStatus: 'healthy' + }) + }) + }) + + describe('projectStatusForApi()', () => { + it('zeros meters on a non-RUNNING row without dropping identity fields', () => { + const projected = projectStatusForApi({ + status: 'FAILED', + cpuUsage: 22, + memoryUsage: 100, + startTime: 1, + operatingDuration: 2, + containerId: 'ctr-keep', + errorMessage: 'boom', + podId: 'pod-keep', + healthStatus: 'unhealthy' + }) + expect(projected).to.include({ + status: 'FAILED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-keep', + errorMessage: 'boom', + podId: 'pod-keep', + healthStatus: 'unhealthy' + }) + }) + + it('does not mutate the original row', () => { + const row = { status: 'STOPPED', cpuUsage: 9, containerId: 'ctr-1' } + const projected = projectStatusForApi(row) + expect(projected.cpuUsage).to.equal(0) + expect(row.cpuUsage).to.equal(9) + }) + + it('clones Sequelize-style rows via get({ plain: true })', () => { + const row = { + get (options) { + expect(options).to.eql({ plain: true }) + return { status: 'STOPPING', cpuUsage: 4, memoryUsage: 8, containerId: 'ctr-2' } + } + } + expect(projectStatusForApi(row)).to.include({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + containerId: 'ctr-2' + }) + }) + + it('always returns lastError, lastErrorAt, and restartCount', () => { + const projected = projectStatusForApi({ + status: 'RUNNING', + cpuUsage: 1, + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + expect(projected).to.include({ + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + }) + + it('fills empty last crash fields when the row omits them', () => { + expect(projectStatusForApi({ status: 'RUNNING', cpuUsage: 1 })).to.include({ + lastError: '', + lastErrorAt: 0, + restartCount: 0 + }) + }) + }) + + describe('applyCrashExtras()', () => { + it('persists lastError, lastErrorAt, and restartCount when present', () => { + const patch = { status: 'RUNNING' } + applyCrashExtras(patch, { + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + expect(patch).to.include({ + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + }) + + it('writes restartCount 0 when lastError is present and restartCount is omitted', () => { + const patch = { status: 'STARTING' } + applyCrashExtras(patch, { + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123 + }) + expect(patch.lastError).to.equal('exitCode=1 oomKilled=false') + expect(patch.lastErrorAt).to.equal(1726660000123) + expect(patch.restartCount).to.equal(0) + }) + + it('leaves last crash fields unset when all extras are omitted', () => { + const patch = { status: 'RUNNING', cpuUsage: 1 } + applyCrashExtras(patch, { status: 'RUNNING' }) + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + + it('does not treat empty lastError as a wipe', () => { + const patch = { status: 'RUNNING' } + applyCrashExtras(patch, { lastError: '', lastErrorAt: 0 }) + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + + it('ignores malformed extras instead of throwing', () => { + const patch = { status: 'RUNNING' } + applyCrashExtras(patch, { lastError: { nested: true }, lastErrorAt: 'nope', restartCount: 'n/a' }) + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + }) +}) From 3e58eae62d0e072108a09d033ec53edb60162093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 19:58:05 +0300 Subject: [PATCH 22/55] Stamp fog lastStatusTime on receipt and store honest microservice snapshots. Use Controller time for liveness, zero non-running meters, reconcile omitted UUIDs, and persist last-crash extras without failing the PUT. --- src/schemas/agent.js | 6 +- src/services/agent-service.js | 86 +++++- test/src/services/agent-service.test.js | 370 +++++++++++++++++++++++- 3 files changed, 449 insertions(+), 13 deletions(-) diff --git a/src/schemas/agent.js b/src/schemas/agent.js index b2a37e9a..10848df3 100644 --- a/src/schemas/agent.js +++ b/src/schemas/agent.js @@ -134,7 +134,11 @@ const microserviceStatus = { ipAddress: { type: 'string' }, ipAddressExternal: { type: 'string' }, execSessionIds: { type: 'array', items: { type: 'string' } }, - podId: { type: 'string' } + podId: { type: 'string' }, + errorMessage: { type: 'string' }, + lastError: { type: 'string' }, + lastErrorAt: { type: 'integer' }, + restartCount: { type: 'integer' } }, required: ['id'], additionalProperties: true diff --git a/src/services/agent-service.js b/src/services/agent-service.js index 1276440b..b81fdbdd 100644 --- a/src/services/agent-service.js +++ b/src/services/agent-service.js @@ -2,7 +2,7 @@ const config = require('../config') // const Sequelize = require('sequelize') const moment = require('moment') // const Op = Sequelize.Op -// const logger = require('../logger') +const logger = require('../logger') const TransactionDecorator = require('../decorators/transaction-decorator') const FogProvisionKeyManager = require('../data/managers/iofog-provision-key-manager') @@ -28,6 +28,12 @@ const MicroserviceService = require('../services/microservices-service') const ApplicationManager = require('../data/managers/application-manager') const constants = require('../helpers/constants') const CatalogContainer = require('../helpers/microservice-container-catalog') +const { + applyCrashExtras, + applyRuntimeMetrics, + coerceRuntimeMetric, + zeroRuntimeMetrics +} = require('../helpers/microservice-runtime-metrics') const SecretManager = require('../data/managers/secret-manager') const ConfigMapManager = require('../data/managers/config-map-manager') const MicroserviceLogStatusManager = require('../data/managers/microservice-log-status-manager') @@ -35,6 +41,8 @@ const MicroserviceExecSessionManager = require('../data/managers/microservice-ex const FogLogStatusManager = require('../data/managers/fog-log-status-manager') const RbacRoleManager = require('../data/managers/rbac-role-manager') +const AGENT_LAST_STATUS_SKEW_MS = 5 * 60 * 1000 + const CHANGE_TRACKING_DEFAULT = {} const CHANGE_TRACKING_KEYS = [ 'config', @@ -277,7 +285,6 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { repositoryCount: agentStatus.repositoryCount, repositoryStatus: agentStatus.repositoryStatus, systemTime: agentStatus.systemTime, - lastStatusTime: agentStatus.lastStatusTime, ipAddress: agentStatus.ipAddress, ipAddressExternal: agentStatus.ipAddressExternal, availableRuntimes: agentStatus.availableRuntimes != null @@ -300,6 +307,15 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { fogStatus = AppHelper.deleteUndefinedFields(fogStatus) + const receiptTime = Date.now() + const agentLastStatusTime = Number(agentStatus.lastStatusTime) + if (Number.isFinite(agentLastStatusTime) && Math.abs(receiptTime - agentLastStatusTime) > AGENT_LAST_STATUS_SKEW_MS) { + logger.warn( + `Agent lastStatusTime ${agentLastStatusTime} differs from Controller receipt time ${receiptTime} for fog ${fog.uuid}` + ) + } + fogStatus.lastStatusTime = receiptTime + const existingFog = await FogManager.findOne({ uuid: fog.uuid }, transaction) @@ -332,31 +348,85 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { } const _updateMicroserviceStatuses = async function (microserviceStatus, fog, transaction) { - for (const status of microserviceStatus) { - let microserviceStatus = { + const reported = Array.isArray(microserviceStatus) ? microserviceStatus : [] + for (const status of reported) { + let observedStatus = { containerId: status.containerId, status: status.status, healthStatus: status.healthStatus, startTime: status.startTime, operatingDuration: status.operatingDuration, - cpuUsage: status.cpuUsage, - memoryUsage: status.memoryUsage, + cpuUsage: coerceRuntimeMetric(status.cpuUsage), + memoryUsage: coerceRuntimeMetric(status.memoryUsage), percentage: status.percentage, errorMessage: status.errorMessage, ipAddress: status.ipAddress, execSessionIds: status.execSessionIds, podId: status.podId } - microserviceStatus = AppHelper.deleteUndefinedFields(microserviceStatus) + applyRuntimeMetrics(observedStatus, observedStatus.status) + applyCrashExtras(observedStatus, status) + observedStatus = AppHelper.deleteUndefinedFields(observedStatus) const microservice = await MicroserviceManager.findOne({ uuid: status.id }, transaction) if (microservice && fog.uuid === microservice.iofogUuid) { await MicroserviceStatusManager.update({ microserviceUuid: status.id - }, microserviceStatus, transaction) + }, observedStatus, transaction) + } + } + await _reconcileOmittedMicroserviceStatuses(reported, fog, transaction) +} + +const _reconcileOmittedMicroserviceStatuses = async function (reportedStatuses, fog, transaction) { + const reportedIds = new Set(reportedStatuses.map((item) => item.id).filter(Boolean)) + const assigned = await MicroserviceManager.findAll({ iofogUuid: fog.uuid }, transaction) + const omitted = (assigned || []).filter((ms) => !reportedIds.has(ms.uuid)) + if (!omitted.length) { + return + } + + const applicationIds = [...new Set(omitted.map((ms) => ms.applicationId).filter((id) => id != null))] + const applications = applicationIds.length + ? await ApplicationManager.findAll({ id: applicationIds }, transaction) + : [] + const applicationById = new Map((applications || []).map((app) => [app.id, app])) + + const stoppedUuids = [] + const unknownUuids = [] + for (const ms of omitted) { + const application = applicationById.get(ms.applicationId) + const desiredInactive = !ms.isActivated || (application && !application.isActivated) + if (desiredInactive) { + stoppedUuids.push(ms.uuid) + } else { + unknownUuids.push(ms.uuid) } } + + const zeros = zeroRuntimeMetrics() + if (stoppedUuids.length) { + await MicroserviceStatusManager.update( + { microserviceUuid: stoppedUuids }, + Object.assign({ status: microserviceState.STOPPED }, zeros), + transaction + ) + } + if (unknownUuids.length) { + await MicroserviceStatusManager.update( + { microserviceUuid: unknownUuids }, + Object.assign({ status: microserviceState.UNKNOWN }, zeros), + transaction + ) + } + + const omittedUuids = stoppedUuids.concat(unknownUuids) + await MicroserviceExecStatusManager.update( + { microserviceUuid: omittedUuids }, + { status: microserviceExecState.INACTIVE, execSessionId: '' }, + transaction + ) } const _mapExtraHost = function (extraHost) { diff --git a/test/src/services/agent-service.test.js b/test/src/services/agent-service.test.js index 197a361f..a1feec4d 100644 --- a/test/src/services/agent-service.test.js +++ b/test/src/services/agent-service.test.js @@ -26,6 +26,7 @@ const FogStates = require('../../../src/enums/fog-state') const constants = require('../../../src/helpers/constants') const config = require('../../../src/config') const Errors = require('../../../src/helpers/errors') +const logger = require('../../../src/logger') global.appRoot = path.resolve(__dirname) @@ -717,6 +718,7 @@ describe('Agent Service', () => { def('platformStatusResponse', () => Promise.resolve(null)) beforeEach(() => { + $sandbox.useFakeTimers({ now: fogStatus.lastStatusTime, toFake: ['Date'] }) $sandbox.stub(Validator, 'validate').returns($validatorResponse) $sandbox.spy(AppHelper, 'deleteUndefinedFields') $sandbox.stub(ioFogManager, 'findOne').returns($findOneResponse) @@ -726,6 +728,7 @@ describe('Agent Service', () => { $sandbox.stub(MicroserviceStatusManager, 'update').returns($updateMicroserviceStatusesResponse) $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').returns($deleteNotRunningResponse) $sandbox.stub(MicroserviceManager, 'findOne').returns($findMicroservice) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) it('calls Validator#validate() with correct args', async () => { @@ -878,10 +881,10 @@ describe('Agent Service', () => { const expectedMicroserviceUpdate = { containerId: microserviceStatus.containerId, status: microserviceStatus.status, - startTime: microserviceStatus.startTime, - operatingDuration: microserviceStatus.operatingDuration, - cpuUsage: microserviceStatus.cpuUsage, - memoryUsage: microserviceStatus.memoryUsage, + startTime: 0, + operatingDuration: 0, + cpuUsage: 0, + memoryUsage: 0, percentage: microserviceStatus.percentage, errorMessage: microserviceStatus.errorMessage, } @@ -980,6 +983,7 @@ describe('Agent Service', () => { def('platformStatusResponse', () => Promise.resolve(null)) beforeEach(() => { + $sandbox.useFakeTimers({ now: fogStatus.lastStatusTime, toFake: ['Date'] }) $sandbox.stub(Validator, 'validate').returns($validatorResponse) $sandbox.spy(AppHelper, 'deleteUndefinedFields') $sandbox.stub(ioFogManager, 'findOne').returns($findOneResponse) @@ -989,6 +993,7 @@ describe('Agent Service', () => { $sandbox.stub(MicroserviceStatusManager, 'update').returns($updateMicroserviceStatusesResponse) $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').returns($deleteNotRunningResponse) $sandbox.stub(MicroserviceManager, 'findOne').returns($findMicroservice) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) it('calls Validator#validate() with correct args', async () => { @@ -1154,6 +1159,7 @@ describe('Agent Service', () => { def('subject', () => AgentService.updateAgentStatus(baseAgentStatus, $fog, transaction)) beforeEach(() => { + $sandbox.stub(logger, 'warn') $sandbox.stub(Validator, 'validate').resolves(true) $sandbox.stub(ioFogManager, 'findOne').resolves({ warningMessage: $existingWarningMessage }) $sandbox.stub(FogPlatformStatusManager, 'getParsedStatus').resolves( @@ -1162,6 +1168,7 @@ describe('Agent Service', () => { $sandbox.stub(ioFogManager, 'update').resolves() $sandbox.stub(JSON, 'parse').returns([]) $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').resolves() + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) it('forces daemonStatus WARNING when platform phase is Progressing', async () => { @@ -1857,6 +1864,7 @@ describe('Agent Service', () => { def('subject', () => $subject.updateAgentStatus(agentStatus, $fog, transaction)) beforeEach(() => { + $sandbox.stub(logger, 'warn') $sandbox.stub(Validator, 'validate').resolves(true) $sandbox.spy(AppHelper, 'deleteUndefinedFields') $sandbox.stub(ioFogManager, 'findOne').resolves({ warningMessage: '' }) @@ -1865,6 +1873,7 @@ describe('Agent Service', () => { $sandbox.stub(MicroserviceStatusManager, 'update').resolves() $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').resolves() $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $uuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) it('persists additive fog status fields and microservice podId', async () => { @@ -1888,6 +1897,359 @@ describe('Agent Service', () => { }) }) + describe('.updateAgentStatus() observed runtime', () => { + const transaction = {} + const receiptTime = 1700000000000 + const fogUuid = 'fog-1' + const reportedUuid = 'ms-reported' + const omittedUuid = 'ms-omitted' + + function buildAgentStatus (microserviceStatusItems, extra = {}) { + return { + daemonStatus: 'RUNNING', + daemonOperatingDuration: 25, + daemonLastStart: 1, + warningMessage: '', + memoryUsage: 1, + diskUsage: 1, + cpuUsage: 1, + memoryViolation: false, + diskViolation: false, + cpuViolation: false, + systemAvailableDisk: 1, + systemAvailableMemory: 1, + systemTotalCpu: 1, + repositoryCount: 0, + repositoryStatus: '[]', + systemTime: 1, + lastStatusTime: 1, + ipAddress: '127.0.0.1', + ipAddressExternal: '127.0.0.1', + lastCommandTime: 1, + tunnelStatus: '{}', + version: '1.1.0', + isReadyToUpgrade: false, + isReadyToRollback: false, + gpsStatus: 'OK', + microserviceStatus: JSON.stringify(microserviceStatusItems), + ...extra, + } + } + + beforeEach(() => { + $sandbox.useFakeTimers({ now: receiptTime, toFake: ['Date'] }) + $sandbox.stub(logger, 'warn') + $sandbox.stub(Validator, 'validate').resolves(true) + $sandbox.stub(ioFogManager, 'findOne').resolves({ warningMessage: '' }) + $sandbox.stub(FogPlatformStatusManager, 'getParsedStatus').resolves(null) + $sandbox.stub(ioFogManager, 'update').resolves() + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').resolves() + $sandbox.stub(ApplicationManager, 'findAll').resolves([]) + }) + + it('stamps fog lastStatusTime with Controller receipt time', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) + + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: fogUuid }, + sinon.match({ lastStatusTime: receiptTime }), + transaction + ) + expect(ioFogManager.update.firstCall.args[1].lastStatusTime).to.not.equal(1) + }) + + it('stamps fog lastStatusTime with Controller receipt time when the agent sends 0', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }], { lastStatusTime: 0 }), + { uuid: fogUuid }, + transaction + ) + + expect(ioFogManager.update.firstCall.args[1].lastStatusTime).to.equal(receiptTime) + }) + + it('stamps fog lastStatusTime with Controller receipt time when the agent clock is hours behind', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus( + [{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }], + { lastStatusTime: receiptTime - (6 * 60 * 60 * 1000) } + ), + { uuid: fogUuid }, + transaction + ) + + expect(ioFogManager.update.firstCall.args[1].lastStatusTime).to.equal(receiptTime) + }) + + it('stores STOPPED meters as 0 when the agent omits cpu/memory', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'STOPPED', containerId: 'ctr-stop' }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-stop', + }), + transaction + ) + }) + + it('stores STOPPED leftover cpu as 0', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'STOPPED', cpuUsage: 12.5, memoryUsage: 64 }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ status: 'STOPPED', cpuUsage: 0, memoryUsage: 0 }), + transaction + ) + }) + + it('stores RUNNING cpu 12.5', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 12.5, memoryUsage: 64 }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ status: 'RUNNING', cpuUsage: 12.5, memoryUsage: 64 }), + transaction + ) + }) + + it('stores lastError, lastErrorAt, and restartCount from the agent', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ + id: reportedUuid, + status: 'RUNNING', + cpuUsage: 1, + errorMessage: '', + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }), + transaction + ) + }) + + it('stores restartCount 0 when lastError is present and restartCount is omitted', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ + id: reportedUuid, + status: 'STARTING', + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123 + }]), + { uuid: fogUuid }, + transaction + ) + + const patch = MicroserviceStatusManager.update.firstCall.args[1] + expect(patch.lastError).to.equal('exitCode=1 oomKilled=false') + expect(patch.lastErrorAt).to.equal(1726660000123) + expect(patch.restartCount).to.equal(0) + }) + + it('does not write last crash fields when the agent omits all extras', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) + + const patch = MicroserviceStatusManager.update.firstCall.args[1] + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + + it('does not fail the PUT when extras are malformed', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ + id: reportedUuid, + status: 'RUNNING', + cpuUsage: 1, + lastError: { nested: true }, + lastErrorAt: 'bogus', + restartCount: 'n/a' + }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledOnce + const patch = MicroserviceStatusManager.update.firstCall.args[1] + expect(patch).to.not.have.property('lastError') + expect(patch.status).to.equal('RUNNING') + }) + + it('treats omitted RUNNING cpu as 0 for this sample', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING' }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ status: 'RUNNING', cpuUsage: 0, memoryUsage: 0 }), + transaction + ) + }) + + it('marks an omitted desired-active microservice UNKNOWN with zeros', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([{ + uuid: omittedUuid, + iofogUuid: fogUuid, + isActivated: true, + applicationId: 7, + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ + status: microserviceState.UNKNOWN, + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + }) + + it('marks an omitted desired-inactive microservice STOPPED with zeros', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([{ + uuid: omittedUuid, + iofogUuid: fogUuid, + isActivated: true, + applicationId: 7, + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: false }]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ + status: microserviceState.STOPPED, + cpuUsage: 0, + memoryUsage: 0, + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + }) + + it('marks an omitted microservice STOPPED when the microservice itself is inactive', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([{ + uuid: omittedUuid, + iofogUuid: fogUuid, + isActivated: false, + applicationId: 7, + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + + await AgentService.updateAgentStatus( + buildAgentStatus([]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ status: microserviceState.STOPPED, cpuUsage: 0 }), + transaction + ) + }) + }) + describe('.deleteNode()', () => { const transaction = {} const error = 'Error!' From 5eda1f1166bce3c818750f12f0027dda6922320c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 19:58:56 +0300 Subject: [PATCH 23/55] Show STOPPING with zero meters when an application is deactivated. Write observed stop immediately so GET does not keep last-running CPU/memory until the next agent report. --- src/services/application-service.js | 14 +++++++ test/src/services/application-service.test.js | 37 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/services/application-service.js b/src/services/application-service.js index c55254e9..72a195b7 100644 --- a/src/services/application-service.js +++ b/src/services/application-service.js @@ -189,6 +189,13 @@ const patchApplicationEndPoint = async function (applicationData, conditions, is } if (oldApplication.isActivated !== applicationData.isActivated) { + if (oldApplication.isActivated && applicationData.isActivated === false) { + const microservices = await ApplicationManager.findApplicationMicroservices(conditions, transaction) + await MicroserviceService.setMicroservicesObservedStopping( + (microservices || []).map((ms) => ms.uuid), + transaction + ) + } await _updateChangeTrackingsAndDeleteMicroservicesByApplicationId(conditions, false, transaction) } } @@ -257,6 +264,13 @@ const updateApplicationEndPoint = async function (applicationData, name, isCLI, await _updateMicroservices(application.name, applicationData.microservices, isCLI, transaction) } if (oldApplication.isActivated !== applicationData.isActivated) { + if (oldApplication.isActivated && applicationData.isActivated === false) { + const microservices = await ApplicationManager.findApplicationMicroservices({ name }, transaction) + await MicroserviceService.setMicroservicesObservedStopping( + (microservices || []).map((ms) => ms.uuid), + transaction + ) + } await _updateChangeTrackingsAndDeleteMicroservicesByApplicationId({ name }, false, transaction) } } diff --git a/test/src/services/application-service.test.js b/test/src/services/application-service.test.js index 6be46142..f3572220 100644 --- a/test/src/services/application-service.test.js +++ b/test/src/services/application-service.test.js @@ -8,6 +8,8 @@ const Validator = require('../../../src/schemas') const ChangeTrackingService = require('../../../src/services/change-tracking-service') const MicroserviceService = require('../../../src/services/microservices-service') const NatsAuthService = require('../../../src/services/nats-auth-service') +const MicroserviceStatusManager = require('../../../src/data/managers/microservice-status-manager') +const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') const Errors = require('../../../src/helpers/errors') const transaction = {} @@ -256,6 +258,41 @@ describe('Application Service', () => { it('rejects rename attempts', () => expect($subject).to.be.rejectedWith('Application Resource Name is immutable')) }) + + context('when deactivating', () => { + const members = [ + { uuid: 'ms-active', iofogUuid: 'fog-1', isActivated: true }, + { uuid: 'ms-already-off', iofogUuid: 'fog-1', isActivated: false } + ] + def('patchData', () => ({ isActivated: false })) + + beforeEach(() => { + $sandbox.stub(ApplicationManager, 'findApplicationMicroservices').resolves(members) + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('sets observed STOPPING and zero meters on all member microservices', async () => { + await $subject + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: ['ms-active', 'ms-already-off'] }, + sinon.match({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: ['ms-active', 'ms-already-off'] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + }) + }) }) describe('.getUserApplicationsEndPoint()', () => { From c6f621cbb57769a37edec4d636d12cd61b6de654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 19:59:21 +0300 Subject: [PATCH 24/55] Return live meters only while RUNNING and stop microservices as STOPPING. Always include last-crash fields on GET, and await not-running microservice deletes on the agent path. --- src/services/microservices-service.js | 51 ++++- .../services/microservices-service.test.js | 183 ++++++++++++++++++ 2 files changed, 226 insertions(+), 8 deletions(-) diff --git a/src/services/microservices-service.js b/src/services/microservices-service.js index c9cfa819..c781c5c5 100644 --- a/src/services/microservices-service.js +++ b/src/services/microservices-service.js @@ -19,6 +19,7 @@ const CatalogItemImageManager = require('../data/managers/catalog-item-image-man const RegistryManager = require('../data/managers/registry-manager') // const RouterManager = require('../data/managers/router-manager') const MicroserviceStates = require('../enums/microservice-state') +const { microserviceState, microserviceExecState } = MicroserviceStates const VolumeMappingManager = require('../data/managers/volume-mapping-manager') const ChangeTrackingService = require('./change-tracking-service') const AppHelper = require('../helpers/app-helper') @@ -51,6 +52,11 @@ const NatsUserRuleManager = require('../data/managers/nats-user-rule-manager') const NatsRuleJwtValidation = require('../helpers/nats-rule-jwt-validation') const NatsInstanceManager = require('../data/managers/nats-instance-manager') const { resolveNatsServerUrl } = require('../helpers/nats-server-url') +const { + applyRuntimeMetrics, + projectStatusForApi, + zeroRuntimeMetrics +} = require('../helpers/microservice-runtime-metrics') const Op = require('sequelize').Op const FogManager = require('../data/managers/iofog-manager') @@ -2032,13 +2038,22 @@ async function deleteMicroserviceEndPoint (microserviceUuid, microserviceData, i async function deleteNotRunningMicroservices (fog, transaction) { const microservices = await MicroserviceManager.findAllWithStatuses({ iofogUuid: fog.uuid }, transaction) - microservices - .filter((microservice) => microservice.delete) - .filter((microservice) => microservice.microserviceStatus.status === MicroserviceStates.UNKNOWN || - microservice.microserviceStatus.status === MicroserviceStates.STOPPING || - microservice.microserviceStatus.status === MicroserviceStates.DELETING || - microservice.microserviceStatus.status === MicroserviceStates.MARKED_FOR_DELETION) - .forEach(async (microservice) => { await deleteMicroserviceWithRoutesAndPortMappings(microservice, transaction) }) + const toDelete = (microservices || []).filter((microservice) => { + if (!microservice.delete) { + return false + } + const status = microservice.microserviceStatus && microservice.microserviceStatus.status + if (!status) { + return false + } + return status === microserviceState.UNKNOWN || + status === microserviceState.STOPPING || + status === microserviceState.DELETING || + status === microserviceState.MARKED_FOR_DELETION + }) + for (const microservice of toDelete) { + await module.exports.deleteMicroserviceWithRoutesAndPortMappings(microservice, transaction) + } } async function createPortMappingEndPoint (microserviceUuid, portMappingData, isCLI, transaction) { @@ -2959,7 +2974,7 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { res.extraHosts = extraHosts.map(eH => ({ name: eH.name, address: eH.template, value: eH.value })) res.images = images.map(i => ({ containerImage: i.containerImage, archId: i.archId })) if (status && status.length) { - res.status = status[0] + res.status = projectStatusForApi(status[0]) } if (execStatus && execStatus.length) { res.execStatus = execStatus[0] @@ -3037,6 +3052,24 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { return res } +async function setMicroservicesObservedStopping (microserviceUuids, transaction) { + const uuids = (microserviceUuids || []).filter(Boolean) + if (!uuids.length) { + return + } + const observedStatus = applyRuntimeMetrics( + Object.assign({ status: microserviceState.STOPPING }, zeroRuntimeMetrics()), + microserviceState.STOPPING + ) + await MicroserviceStatusManager.update({ microserviceUuid: uuids }, observedStatus, transaction) + await MicroserviceExecStatusManager.update({ + microserviceUuid: uuids + }, { + status: microserviceExecState.INACTIVE, + execSessionId: '' + }, transaction) +} + async function startMicroserviceEndPoint (microserviceUuid, isCLI, transaction) { const microservice = await MicroserviceManager.findOneWithCategory({ uuid: microserviceUuid }, transaction) if (!microservice) { @@ -3071,6 +3104,7 @@ async function stopMicroserviceEndPoint (microserviceUuid, isCLI, transaction) { } await MicroserviceManager.update({ uuid: microservice.uuid }, { isActivated: false }, transaction) + await setMicroservicesObservedStopping([microservice.uuid], transaction) await ChangeTrackingService.update(microservice.iofogUuid, ChangeTrackingService.events.microserviceList, transaction) return { @@ -3136,6 +3170,7 @@ module.exports = { updateChangeTracking: _updateChangeTracking, startMicroserviceEndPoint: TransactionDecorator.generateTransaction(startMicroserviceEndPoint), stopMicroserviceEndPoint: TransactionDecorator.generateTransaction(stopMicroserviceEndPoint), + setMicroservicesObservedStopping, reconcileNatsForApplication: TransactionDecorator.generateTransaction(reconcileNatsForApplication), injectServiceAccountVolume: _injectServiceAccountVolume, stripUserServiceAccountVolumeMappings: _stripUserServiceAccountVolumeMappings, diff --git a/test/src/services/microservices-service.test.js b/test/src/services/microservices-service.test.js index 928e0512..ed2ed71b 100644 --- a/test/src/services/microservices-service.test.js +++ b/test/src/services/microservices-service.test.js @@ -1156,4 +1156,187 @@ describe('Microservices Service', () => { ) }) }) + + describe('.buildGetMicroserviceResponse() observed status', () => { + def('subject', () => $service.buildGetMicroserviceResponse($microservice, transaction)) + def('microservice', () => ({ + uuid: 'msvc-uuid', + name: 'test-msvc', + applicationId: 42, + logSize: 1024 + })) + + beforeEach(() => { + stubBuildGetResponseDeps($sandbox) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ name: 'my-app' }) + }) + + it('zeros CPU when status is STOPPED', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'STOPPED', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10, + containerId: 'ctr-keep' + }]) + + const result = await $subject + expect(result.status).to.include({ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-keep' + }) + }) + + it('zeros CPU when status is STOPPING', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'STOPPING', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10, + containerId: 'ctr-keep' + }]) + + const result = await $subject + expect(result.status).to.include({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-keep' + }) + }) + + it('keeps RUNNING meters', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'RUNNING', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10 + }]) + + const result = await $subject + expect(result.status).to.include({ + status: 'RUNNING', + cpuUsage: 12.5, + memoryUsage: 64 + }) + }) + + it('returns lastError, lastErrorAt, and restartCount on GET', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'RUNNING', + cpuUsage: 1, + errorMessage: '', + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }]) + + const result = await $subject + expect(result.status).to.include({ + errorMessage: '', + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + }) + + it('returns empty last crash fields when the row has none', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0 + }]) + + const result = await $subject + expect(result.status).to.include({ + lastError: '', + lastErrorAt: 0, + restartCount: 0 + }) + }) + }) + + describe('.stopMicroserviceEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const microservice = buildMicroserviceRecord({ + uuid: msvcUuid, + catalogItem: { category: 'USER' } + }) + + def('subject', () => $service.stopMicroserviceEndPoint(msvcUuid, isCLI, transaction)) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(microservice) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('sets observed STOPPING with zero meters and inactive exec', async () => { + const result = await $subject + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { isActivated: false }, + transaction + ) + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [msvcUuid] }, + sinon.match({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [msvcUuid] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + expect(result).to.eql({ uuid: msvcUuid, isActivated: false }) + }) + }) + + describe('.deleteNotRunningMicroservices()', () => { + def('subject', () => $service.deleteNotRunningMicroservices({ uuid: 'fog-uuid' }, transaction)) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses') + $sandbox.stub($service, 'deleteMicroserviceWithRoutesAndPortMappings').resolves() + }) + + it('awaits deletes for not-running flagged microservices', async () => { + MicroserviceManager.findAllWithStatuses.resolves([ + { delete: true, microserviceStatus: { status: 'UNKNOWN' } }, + { delete: true, microserviceStatus: { status: 'STOPPING' } }, + { delete: true, microserviceStatus: { status: 'RUNNING' } }, + { delete: false, microserviceStatus: { status: 'DELETING' } }, + { delete: true, microserviceStatus: null } + ]) + + await $subject + + expect($service.deleteMicroserviceWithRoutesAndPortMappings).to.have.been.calledTwice + expect($service.deleteMicroserviceWithRoutesAndPortMappings.firstCall).to.have.been.calledWith( + sinon.match({ microserviceStatus: { status: 'UNKNOWN' } }), + transaction + ) + expect($service.deleteMicroserviceWithRoutesAndPortMappings.secondCall).to.have.been.calledWith( + sinon.match({ microserviceStatus: { status: 'STOPPING' } }), + transaction + ) + }) + }) }) From 1b25b52e53d9820ab0e3b1a5cd229cdeff5ed05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 19:59:58 +0300 Subject: [PATCH 25/55] Add a chunk size setting for fog status liveness. Walk quiet nodes in small batches so the cleaner does not hold one fleet-wide write. --- src/config/config.yaml | 1 + src/config/env-mapping.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/config/config.yaml b/src/config/config.yaml index 110fb8b3..f13eb8a6 100644 --- a/src/config/config.yaml +++ b/src/config/config.yaml @@ -75,6 +75,7 @@ settings: # defaultJobInterval: 120 # Default job interval in seconds fogStatusUpdateInterval: 30 # Fog status update interval in seconds fogStatusUpdateTolerance: 3 # Fog status update tolerance + fogStatusLivenessChunkSize: 50 # Fogs processed per quiet-node liveness transaction (default: 50) fogExpiredTokenCleanupInterval: 300 # Fog expired token cleanup interval in seconds eventRetentionDays: 7 # Days to retain events (default: 7) eventCleanupInterval: 86400 # Cleanup job interval in seconds (default: 24 hours) diff --git a/src/config/env-mapping.js b/src/config/env-mapping.js index 870f00cc..4d7069c0 100644 --- a/src/config/env-mapping.js +++ b/src/config/env-mapping.js @@ -58,6 +58,7 @@ module.exports = { // Settings Configuration FOG_STATUS_UPDATE_INTERVAL: 'settings.fogStatusUpdateInterval', FOG_STATUS_UPDATE_TOLERANCE: 'settings.fogStatusUpdateTolerance', + FOG_STATUS_LIVENESS_CHUNK_SIZE: 'settings.fogStatusLivenessChunkSize', FOG_EXPIRED_TOKEN_CLEANUP_INTERVAL: 'settings.fogExpiredTokenCleanupInterval', EVENT_RETENTION_DAYS: 'settings.eventRetentionDays', EVENT_CLEANUP_INTERVAL: 'settings.eventCleanupInterval', From 1974d9e258f478e678888261f0dd2bf9fbe67cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 20:00:04 +0300 Subject: [PATCH 26/55] Select quiet fogs in small lastStatusTime batches. Give the liveness cleaner a cursor query so it can update nodes without loading the whole fleet. --- src/data/managers/iofog-manager.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/data/managers/iofog-manager.js b/src/data/managers/iofog-manager.js index db5b7ed0..bb7a45f7 100644 --- a/src/data/managers/iofog-manager.js +++ b/src/data/managers/iofog-manager.js @@ -76,6 +76,25 @@ class FogManager extends BaseManager { return rows.map((row) => row.uuid) } + async findStatusLivenessCandidates ({ afterUuid, limit, heardBefore, daemonStatuses }, transaction) { + const where = { + daemonStatus: { [Op.in]: daemonStatuses } + } + if (heardBefore != null) { + where.lastStatusTime = { [Op.lt]: heardBefore } + } + if (afterUuid) { + where.uuid = { [Op.gt]: afterUuid } + } + return Fog.findAll({ + attributes: ['uuid', 'statusFrequency', 'lastStatusTime', 'daemonStatus'], + where, + order: [['uuid', 'ASC']], + limit, + transaction + }) + } + updateLastActive (uuid, timestamp, transaction) { return Fog.update({ lastActive: timestamp From 2f2d988941113ece2db69dcbb29bd994460c262f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 20:00:09 +0300 Subject: [PATCH 27/55] Mark quiet fogs UNKNOWN without deleting microservices. Clamp status frequency, skip overlapping runs, and leave already-stopped workloads alone. --- src/jobs/fog-status-job.js | 193 ++++++++++++++++++++++++------------- 1 file changed, 126 insertions(+), 67 deletions(-) diff --git a/src/jobs/fog-status-job.js b/src/jobs/fog-status-job.js index 118c0143..d42b896a 100644 --- a/src/jobs/fog-status-job.js +++ b/src/jobs/fog-status-job.js @@ -1,102 +1,161 @@ +const { Op } = require('sequelize') const TransactionDecorator = require('../decorators/transaction-decorator') const { PRIORITY_BACKGROUND } = require('../helpers/transaction-runner') const FogManager = require('../data/managers/iofog-manager') +const ApplicationManager = require('../data/managers/application-manager') const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') -const MicroserviceService = require('../services/microservices-service') const { microserviceState, microserviceExecState } = require('../enums/microservice-state') const FogStates = require('../enums/fog-state') +const { zeroRuntimeMetrics } = require('../helpers/microservice-runtime-metrics') const Config = require('../config') const logger = require('../logger') +const MIN_FOG_STATUS_FREQUENCY_SEC = 10 +const LIVENESS_DAEMON_STATUSES = [FogStates.RUNNING, FogStates.WARNING] const scheduleTime = Config.get('settings.fogStatusUpdateInterval') * 1000 +let inFlight = false + +function clampedStatusFrequencySec (statusFrequency) { + const numeric = Number(statusFrequency) + const freqSec = Number.isFinite(numeric) ? numeric : 0 + return Math.max(freqSec, MIN_FOG_STATUS_FREQUENCY_SEC) +} + +function isFogStatusStale (fog, tolerance, nowMs) { + const deadlineMs = fog.lastStatusTime + clampedStatusFrequencySec(fog.statusFrequency) * 1000 * tolerance + return nowMs > deadlineMs +} + +function isDesiredInactive (microservice, application) { + return microservice.isActivated === false || (application && application.isActivated === false) +} + +function shouldMarkUnknownOnQuietFog (microservice, application) { + if (!microservice.microserviceStatus) { + return false + } + if (isDesiredInactive(microservice, application)) { + return false + } + return true +} + async function run () { + if (inFlight) { + return + } + inFlight = true try { - const _updateFogsConnectionStatus = TransactionDecorator.generateTransaction( - updateFogsConnectionStatus, - { priority: PRIORITY_BACKGROUND, label: 'fogStatus.updateConnection' } - ) - await _updateFogsConnectionStatus() + await module.exports.runLivenessPass() } catch (error) { logger.error('Error during fog status update:', error) } finally { + inFlight = false setTimeout(run, scheduleTime) } } -async function updateFogsConnectionStatus (transaction) { - const unknownFogUuids = await _updateFogStatus(transaction) - const microservices = await _updateMicroserviceStatus(unknownFogUuids, transaction) - await _deleteNotRunningMicroservices(microservices, transaction) -} +async function runLivenessPass () { + const chunkSize = Math.max(1, Number(Config.get('settings.fogStatusLivenessChunkSize', 50)) || 50) + const tolerance = Config.get('settings.fogStatusUpdateTolerance') + const processChunk = TransactionDecorator.generateTransaction( + processLivenessChunk, + { priority: PRIORITY_BACKGROUND, label: 'fogStatus.updateConnection' } + ) -async function _updateFogStatus (transaction) { - const statusUpdateTolerance = Config.get('settings.fogStatusUpdateTolerance') - const fogs = [ - ...await FogManager.findAll({ daemonStatus: FogStates.RUNNING }, transaction), - ...await FogManager.findAll({ daemonStatus: FogStates.WARNING }, transaction) - ] - const unknownFogUuids = fogs - .filter((fog) => { - const statusUpdateToleranceMs = fog.statusFrequency * 1000 * statusUpdateTolerance - return (Date.now() - fog.lastStatusTime) > statusUpdateToleranceMs - }) - .map((fog) => fog.uuid) - - const where = { uuid: unknownFogUuids } - const data = { daemonStatus: FogStates.UNKNOWN } - await FogManager.update(where, data, transaction) - return unknownFogUuids + let afterUuid = null + while (true) { + const result = await processChunk(afterUuid, chunkSize, tolerance) + if (!result || !result.nextCursor) { + break + } + afterUuid = result.nextCursor + } } -async function _updateMicroserviceStatus (unknownFogUuids, transaction) { - const microservices = await MicroserviceManager.findAllWithStatuses({ iofogUuid: unknownFogUuids }, transaction) - - // Filter out inactive microservices that are already STOPPED - they should not be updated to UNKNOWN - const microserviceStatusIds = microservices - .filter((microservice) => { - // Skip if no microservice status - if (!microservice.microserviceStatus) { - return false - } - // Skip if microservice is inactive and already STOPPED - if (!microservice.isActivated && microservice.microserviceStatus.status === microserviceState.STOPPED) { - return false - } - return true - }) - .map((microservice) => microservice.microserviceStatus.id) - - const microserviceExecStatusIds = microservices - .filter((microservice) => { - // Skip if no exec status - if (!microservice.microserviceExecStatus) { - return false - } - // Skip if microservice is inactive and already STOPPED - if (!microservice.isActivated && microservice.microserviceStatus && microservice.microserviceStatus.status === microserviceState.STOPPED) { - return false - } - return true - }) - .map((microservice) => microservice.microserviceExecStatus.id) - - await MicroserviceStatusManager.update({ id: microserviceStatusIds }, { status: microserviceState.UNKNOWN }, transaction) - await MicroserviceExecStatusManager.update({ id: microserviceExecStatusIds }, { execSesssionId: '', status: microserviceExecState.INACTIVE }, transaction) - return microservices +async function processLivenessChunk (afterUuid, chunkSize, tolerance, transaction) { + const nowMs = Date.now() + const heardBefore = nowMs - MIN_FOG_STATUS_FREQUENCY_SEC * 1000 * tolerance + const candidates = await FogManager.findStatusLivenessCandidates({ + afterUuid, + limit: chunkSize, + heardBefore, + daemonStatuses: LIVENESS_DAEMON_STATUSES + }, transaction) + + if (!candidates.length) { + return { nextCursor: null, staleCount: 0 } + } + + const nextCursor = candidates[candidates.length - 1].uuid + const staleFogs = candidates.filter((fog) => isFogStatusStale(fog, tolerance, nowMs)) + if (!staleFogs.length) { + return { nextCursor, staleCount: 0 } + } + + const unknownFogUuids = staleFogs.map((fog) => fog.uuid) + await FogManager.update({ uuid: unknownFogUuids }, { daemonStatus: FogStates.UNKNOWN }, transaction) + await markDesiredActiveUnknown(unknownFogUuids, transaction) + return { nextCursor, staleCount: staleFogs.length } } -async function _deleteNotRunningMicroservices (microservices, transaction) { +async function markDesiredActiveUnknown (unknownFogUuids, transaction) { + if (!unknownFogUuids.length) { + return + } + + const microservices = await MicroserviceManager.findAllWithStatuses({ + iofogUuid: { [Op.in]: unknownFogUuids } + }, transaction) + if (!microservices.length) { + return + } + + const applicationIds = [...new Set(microservices.map((ms) => ms.applicationId).filter((id) => id != null))] + const applications = applicationIds.length + ? await ApplicationManager.findAll({ id: applicationIds }, transaction) + : [] + const applicationById = new Map((applications || []).map((app) => [app.id, app])) + + const statusIds = [] + const execIds = [] for (const microservice of microservices) { - if (microservice.delete) { - await MicroserviceService.deleteMicroserviceWithRoutesAndPortMappings(microservice, transaction) + const application = applicationById.get(microservice.applicationId) + if (!shouldMarkUnknownOnQuietFog(microservice, application)) { + continue } + statusIds.push(microservice.microserviceStatus.id) + if (microservice.microserviceExecStatus) { + execIds.push(microservice.microserviceExecStatus.id) + } + } + + if (statusIds.length) { + await MicroserviceStatusManager.update( + { id: statusIds }, + Object.assign({ status: microserviceState.UNKNOWN }, zeroRuntimeMetrics()), + transaction + ) + } + if (execIds.length) { + await MicroserviceExecStatusManager.update( + { id: execIds }, + { execSessionId: '', status: microserviceExecState.INACTIVE }, + transaction + ) } } module.exports = { - run + run, + runLivenessPass, + processLivenessChunk, + isFogStatusStale, + shouldMarkUnknownOnQuietFog, + clampedStatusFrequencySec, + MIN_FOG_STATUS_FREQUENCY_SEC } From e89e53a9c8a33ff21fd10582c5585a0044444e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 20:00:18 +0300 Subject: [PATCH 28/55] Force STOPPED with zero meters for leftover inactive workloads. Rewrite stale RUNNING or STOPPING status when the microservice or application is already deactivated. --- src/jobs/stopped-app-status-job.js | 108 ++++++++++++++++++----------- 1 file changed, 67 insertions(+), 41 deletions(-) diff --git a/src/jobs/stopped-app-status-job.js b/src/jobs/stopped-app-status-job.js index 00d752c7..7346464b 100644 --- a/src/jobs/stopped-app-status-job.js +++ b/src/jobs/stopped-app-status-job.js @@ -1,3 +1,4 @@ +const { Op } = require('sequelize') const TransactionDecorator = require('../decorators/transaction-decorator') const { PRIORITY_BACKGROUND } = require('../helpers/transaction-runner') @@ -5,6 +6,7 @@ const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const { microserviceState, microserviceExecState } = require('../enums/microservice-state') +const { zeroRuntimeMetrics } = require('../helpers/microservice-runtime-metrics') const Config = require('../config') const ApplicationManager = require('../data/managers/application-manager') @@ -12,55 +14,68 @@ const logger = require('../logger') const scheduleTime = Config.get('settings.fogStatusUpdateInterval') * 1000 +const STALE_DESIRED_INACTIVE_STATUSES = new Set([ + microserviceState.DELETED, + microserviceState.DELETING, + microserviceState.RUNNING, + microserviceState.STOPPING +]) + +let inFlight = false + +function shouldForceObservedStopped (microservice) { + const status = microservice.microserviceStatus && microservice.microserviceStatus.status + return STALE_DESIRED_INACTIVE_STATUSES.has(status) +} + async function run () { + if (inFlight) { + return + } + inFlight = true try { - const _updateStoppedApplicationMicroserviceStatus = TransactionDecorator.generateTransaction( - updateStoppedApplicationMicroserviceStatus, - { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.application' } - ) - const _updateStoppedMicroserviceStatus = TransactionDecorator.generateTransaction( - updateStoppedMicroserviceStatus, - { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.microservice' } - ) - - // Handle microservices from deactivated applications - await _updateStoppedApplicationMicroserviceStatus() - // Handle individually deactivated microservices - await _updateStoppedMicroserviceStatus() + await module.exports.runSafetyNetPass() } catch (error) { logger.error('Error during stopped application status update:', error) } finally { + inFlight = false setTimeout(run, scheduleTime) } } -async function updateStoppedApplicationMicroserviceStatus (transaction) { - // Get all deactivated applications +async function runSafetyNetPass () { + const updateStoppedApplicationMicroserviceStatus = TransactionDecorator.generateTransaction( + updateApplicationMicroservices, + { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.application' } + ) + const updateStoppedMicroserviceStatus = TransactionDecorator.generateTransaction( + updateDeactivatedMicroservices, + { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.microservice' } + ) + + await updateStoppedApplicationMicroserviceStatus() + await updateStoppedMicroserviceStatus() +} + +async function updateApplicationMicroservices (transaction) { const stoppedApplications = await ApplicationManager.findAllWithAttributes({ isActivated: false }, ['id'], transaction) if (stoppedApplications.length === 0) { return } - // Get all microservices from these applications const applicationIds = stoppedApplications.map(app => app.id) - const { Op } = require('sequelize') const stoppedMicroservices = await MicroserviceManager.findAllWithStatuses({ applicationId: { [Op.in]: applicationIds } }, transaction) - await _updateMicroserviceStatusStopped(stoppedMicroservices, transaction) + await updateMicroserviceStatusStopped(stoppedMicroservices, transaction) } -async function updateStoppedMicroserviceStatus (transaction) { - // Get all individually deactivated microservices (where microservice isActivated = false but parent application is still active) - const { Op } = require('sequelize') - - // First get all active applications +async function updateDeactivatedMicroservices (transaction) { const activeApplications = await ApplicationManager.findAllWithAttributes({ isActivated: true }, ['id'], transaction) if (activeApplications.length === 0) { return } - // Then get microservices that are individually deactivated but belong to active applications const activeApplicationIds = activeApplications.map(app => app.id) const stoppedMicroservices = await MicroserviceManager.findAllWithStatuses({ isActivated: false, @@ -71,27 +86,38 @@ async function updateStoppedMicroserviceStatus (transaction) { return } - await _updateMicroserviceStatusStopped(stoppedMicroservices, transaction) + await updateMicroserviceStatusStopped(stoppedMicroservices, transaction) } -async function _updateMicroserviceStatusStopped (stoppedMicroservices, transaction) { - const microserviceStatusIds = stoppedMicroservices - .filter((microservice) => microservice.microserviceStatus && (microservice.microserviceStatus.status === microserviceState.DELETED || - microservice.microserviceStatus.status === microserviceState.DELETING)) - .map((microservice) => microservice.microserviceStatus.id) - const microserviceExecStatusIds = stoppedMicroservices - .filter((microservice) => - microservice.microserviceStatus && - (microservice.microserviceStatus.status === microserviceState.DELETED || - microservice.microserviceStatus.status === microserviceState.DELETING) && - microservice.microserviceExecStatus - ) +async function updateMicroserviceStatusStopped (stoppedMicroservices, transaction) { + const toStop = (stoppedMicroservices || []).filter(shouldForceObservedStopped) + const microserviceStatusIds = toStop.map((microservice) => microservice.microserviceStatus.id) + const microserviceExecStatusIds = toStop + .filter((microservice) => microservice.microserviceExecStatus) .map((microservice) => microservice.microserviceExecStatus.id) - await MicroserviceStatusManager.update({ id: microserviceStatusIds }, { status: microserviceState.STOPPED }, transaction) - await MicroserviceExecStatusManager.update({ id: microserviceExecStatusIds }, { execSesssionId: '', status: microserviceExecState.INACTIVE }, transaction) - return stoppedMicroservices + + if (microserviceStatusIds.length) { + await MicroserviceStatusManager.update( + { id: microserviceStatusIds }, + Object.assign({ status: microserviceState.STOPPED }, zeroRuntimeMetrics()), + transaction + ) + } + if (microserviceExecStatusIds.length) { + await MicroserviceExecStatusManager.update( + { id: microserviceExecStatusIds }, + { execSessionId: '', status: microserviceExecState.INACTIVE }, + transaction + ) + } + return toStop } module.exports = { - run + run, + runSafetyNetPass, + updateApplicationMicroservices, + updateDeactivatedMicroservices, + updateMicroserviceStatusStopped, + shouldForceObservedStopped } From 2191f05b6453f99d6a8483cb5bd7a38acb767a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 20:00:36 +0300 Subject: [PATCH 29/55] Cover fog liveness and stopped-app status jobs. Lock in the frequency floor, empty-list skip, overlap no-op, and no deletes from the liveness cleaner. --- test/src/jobs/fog-status-job.test.js | 203 +++++++++++++++++++ test/src/jobs/stopped-app-status-job.test.js | 122 +++++++++++ 2 files changed, 325 insertions(+) create mode 100644 test/src/jobs/fog-status-job.test.js create mode 100644 test/src/jobs/stopped-app-status-job.test.js diff --git a/test/src/jobs/fog-status-job.test.js b/test/src/jobs/fog-status-job.test.js new file mode 100644 index 00000000..76139cf3 --- /dev/null +++ b/test/src/jobs/fog-status-job.test.js @@ -0,0 +1,203 @@ +const { expect } = require('chai') +const fs = require('fs') +const path = require('path') +const sinon = require('sinon') + +const FogManager = require('../../../src/data/managers/iofog-manager') +const ApplicationManager = require('../../../src/data/managers/application-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceStatusManager = require('../../../src/data/managers/microservice-status-manager') +const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const FogStatusJob = require('../../../src/jobs/fog-status-job') +const FogStates = require('../../../src/enums/fog-state') + +const JOB_SOURCE = fs.readFileSync(path.join(__dirname, '../../../src/jobs/fog-status-job.js'), 'utf8') + +describe('fog-status-job', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => $sandbox.restore()) + + describe('clampedStatusFrequencySec()', () => { + it('uses a 10s floor when frequency is 0', () => { + expect(FogStatusJob.clampedStatusFrequencySec(0)).to.equal(10) + }) + + it('keeps a frequency above the floor', () => { + expect(FogStatusJob.clampedStatusFrequencySec(30)).to.equal(30) + }) + }) + + describe('isFogStatusStale()', () => { + it('does not treat frequency 0 as an immediate miss', () => { + const nowMs = 1_000_000 + const fog = { lastStatusTime: nowMs - 15_000, statusFrequency: 0 } + expect(FogStatusJob.isFogStatusStale(fog, 3, nowMs)).to.equal(false) + }) + + it('marks a fog stale after the clamped deadline', () => { + const nowMs = 1_000_000 + const fog = { lastStatusTime: nowMs - 31_000, statusFrequency: 0 } + expect(FogStatusJob.isFogStatusStale(fog, 3, nowMs)).to.equal(true) + }) + + it('keeps a just-heard fog live even when an agent clock of 0 would miss the deadline', () => { + const nowMs = 1_700_000_000_000 + expect(FogStatusJob.isFogStatusStale({ lastStatusTime: nowMs, statusFrequency: 10 }, 3, nowMs)).to.equal(false) + expect(FogStatusJob.isFogStatusStale({ lastStatusTime: 0, statusFrequency: 10 }, 3, nowMs)).to.equal(true) + }) + }) + + describe('shouldMarkUnknownOnQuietFog()', () => { + it('skips desired-inactive STOPPED microservices', () => { + expect(FogStatusJob.shouldMarkUnknownOnQuietFog({ + isActivated: false, + microserviceStatus: { id: 1, status: 'STOPPED' } + }, { isActivated: true })).to.equal(false) + }) + + it('skips microservices on a deactivated application', () => { + expect(FogStatusJob.shouldMarkUnknownOnQuietFog({ + isActivated: true, + microserviceStatus: { id: 1, status: 'RUNNING' } + }, { isActivated: false })).to.equal(false) + }) + + it('marks desired-active workloads unknown', () => { + expect(FogStatusJob.shouldMarkUnknownOnQuietFog({ + isActivated: true, + microserviceStatus: { id: 1, status: 'RUNNING' } + }, { isActivated: true })).to.equal(true) + }) + }) + + describe('processLivenessChunk()', () => { + beforeEach(() => { + $sandbox.stub(FogManager, 'findStatusLivenessCandidates') + $sandbox.stub(FogManager, 'update') + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses') + $sandbox.stub(ApplicationManager, 'findAll') + $sandbox.stub(MicroserviceStatusManager, 'update') + $sandbox.stub(MicroserviceExecStatusManager, 'update') + }) + + it('does not update when no quiet fogs are found', async () => { + FogManager.findStatusLivenessCandidates.resolves([]) + + const result = await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(result).to.eql({ nextCursor: null, staleCount: 0 }) + expect(FogManager.update).to.not.have.been.called + expect(MicroserviceManager.findAllWithStatuses).to.not.have.been.called + }) + + it('does not update when candidates are still within their deadline', async () => { + const nowMs = Date.now() + FogManager.findStatusLivenessCandidates.resolves([{ + uuid: 'fog-fresh', + statusFrequency: 0, + lastStatusTime: nowMs - 1_000, + daemonStatus: FogStates.RUNNING + }]) + + const result = await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(result.nextCursor).to.equal('fog-fresh') + expect(result.staleCount).to.equal(0) + expect(FogManager.update).to.not.have.been.called + expect(MicroserviceManager.findAllWithStatuses).to.not.have.been.called + }) + + it('marks a quiet desired-active microservice UNKNOWN with zero meters', async () => { + const nowMs = Date.now() + FogManager.findStatusLivenessCandidates.resolves([{ + uuid: 'fog-quiet', + statusFrequency: 10, + lastStatusTime: nowMs - 60_000, + daemonStatus: FogStates.RUNNING + }]) + MicroserviceManager.findAllWithStatuses.resolves([{ + applicationId: 7, + isActivated: true, + microserviceStatus: { id: 11, status: 'RUNNING' }, + microserviceExecStatus: { id: 21 } + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + FogManager.update.resolves() + MicroserviceStatusManager.update.resolves() + MicroserviceExecStatusManager.update.resolves() + + const result = await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(result.staleCount).to.equal(1) + expect(FogManager.update).to.have.been.calledWith( + { uuid: ['fog-quiet'] }, + { daemonStatus: FogStates.UNKNOWN }, + transaction + ) + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { id: [11] }, + sinon.match({ + status: 'UNKNOWN', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { id: [21] }, + { execSessionId: '', status: 'INACTIVE' }, + transaction + ) + }) + + it('does not mark desired-inactive STOPPED microservices UNKNOWN', async () => { + const nowMs = Date.now() + FogManager.findStatusLivenessCandidates.resolves([{ + uuid: 'fog-quiet', + statusFrequency: 10, + lastStatusTime: nowMs - 60_000, + daemonStatus: FogStates.WARNING + }]) + MicroserviceManager.findAllWithStatuses.resolves([{ + applicationId: 7, + isActivated: false, + microserviceStatus: { id: 11, status: 'STOPPED' }, + microserviceExecStatus: { id: 21 } + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + FogManager.update.resolves() + + await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(FogManager.update).to.have.been.calledOnce + expect(MicroserviceStatusManager.update).to.not.have.been.called + expect(MicroserviceExecStatusManager.update).to.not.have.been.called + }) + }) + + describe('run()', () => { + it('runs inner work once when a second pass overlaps', async () => { + $sandbox.useFakeTimers() + let release + const blocked = new Promise((resolve) => { release = resolve }) + $sandbox.stub(FogStatusJob, 'runLivenessPass').callsFake(() => blocked) + + const first = FogStatusJob.run() + await Promise.resolve() + await FogStatusJob.run() + expect(FogStatusJob.runLivenessPass).to.have.been.calledOnce + + release() + await first + }) + }) + + it('does not delete microservices from the liveness job', () => { + expect(JOB_SOURCE).to.not.include('deleteMicroserviceWithRoutesAndPortMappings') + expect(JOB_SOURCE).to.not.include('deleteNotRunningMicroservices') + }) +}) diff --git a/test/src/jobs/stopped-app-status-job.test.js b/test/src/jobs/stopped-app-status-job.test.js new file mode 100644 index 00000000..1404ac76 --- /dev/null +++ b/test/src/jobs/stopped-app-status-job.test.js @@ -0,0 +1,122 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const ApplicationManager = require('../../../src/data/managers/application-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceStatusManager = require('../../../src/data/managers/microservice-status-manager') +const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const StoppedAppStatusJob = require('../../../src/jobs/stopped-app-status-job') + +describe('stopped-app-status-job', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => $sandbox.restore()) + + describe('shouldForceObservedStopped()', () => { + it('matches leftover observed statuses on desired-inactive workloads', () => { + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'RUNNING' } + })).to.equal(true) + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'STOPPING' } + })).to.equal(true) + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'DELETED' } + })).to.equal(true) + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'DELETING' } + })).to.equal(true) + }) + + it('leaves already STOPPED rows alone', () => { + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'STOPPED' } + })).to.equal(false) + }) + }) + + describe('updateMicroserviceStatusStopped()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + }) + + it('does not update when there are no leftover statuses', async () => { + await StoppedAppStatusJob.updateMicroserviceStatusStopped([{ + microserviceStatus: { id: 1, status: 'STOPPED' }, + microserviceExecStatus: { id: 2 } + }], transaction) + + expect(MicroserviceStatusManager.update).to.not.have.been.called + expect(MicroserviceExecStatusManager.update).to.not.have.been.called + }) + + it('sets desired-inactive RUNNING to STOPPED with zero meters', async () => { + await StoppedAppStatusJob.updateMicroserviceStatusStopped([{ + isActivated: false, + microserviceStatus: { id: 11, status: 'RUNNING' }, + microserviceExecStatus: { id: 21 } + }], transaction) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { id: [11] }, + sinon.match({ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { id: [21] }, + { execSessionId: '', status: 'INACTIVE' }, + transaction + ) + }) + }) + + describe('updateDeactivatedMicroservices()', () => { + it('skips status updates when no individually deactivated microservices exist', async () => { + $sandbox.stub(ApplicationManager, 'findAllWithAttributes').resolves([{ id: 3 }]) + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses').resolves([]) + $sandbox.stub(MicroserviceStatusManager, 'update') + + await StoppedAppStatusJob.updateDeactivatedMicroservices(transaction) + + expect(MicroserviceStatusManager.update).to.not.have.been.called + }) + }) + + describe('updateApplicationMicroservices()', () => { + it('skips status updates when no deactivated applications exist', async () => { + $sandbox.stub(ApplicationManager, 'findAllWithAttributes').resolves([]) + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses') + $sandbox.stub(MicroserviceStatusManager, 'update') + + await StoppedAppStatusJob.updateApplicationMicroservices(transaction) + + expect(MicroserviceManager.findAllWithStatuses).to.not.have.been.called + expect(MicroserviceStatusManager.update).to.not.have.been.called + }) + }) + + describe('run()', () => { + it('runs inner work once when a second pass overlaps', async () => { + $sandbox.useFakeTimers() + let release + const blocked = new Promise((resolve) => { release = resolve }) + $sandbox.stub(StoppedAppStatusJob, 'runSafetyNetPass').callsFake(() => blocked) + + const first = StoppedAppStatusJob.run() + await Promise.resolve() + await StoppedAppStatusJob.run() + expect(StoppedAppStatusJob.runSafetyNetPass).to.have.been.calledOnce + + release() + await first + }) + }) +}) From 582744a2c2d1aa0c6a6f5e01a3a8c8de6c81fcbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 20:00:49 +0300 Subject: [PATCH 30/55] Document receipt-time lastStatusTime, live-only meters, and last-crash GET fields. Describe the liveness cleaner versus the stopped-app safety net, and record the v3.9.0-rc.3 notes. --- CHANGELOG.md | 21 +++++++++++++++++++++ docs/architecture.md | 4 ++-- docs/swagger.yaml | 45 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 890aad79..fc22d333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [v3.9.0-rc.3] + +Observed microservice status and fog liveness: live CPU/memory only while a microservice is RUNNING, and node quietness judged by when Controller last received a status PUT. + +### Fixed + +- **Stopped/inactive microservices** no longer keep the last CPU/memory sample after stop, deactivate, or a non-RUNNING agent report. +- **Fog liveness** uses Controller receipt time (`lastStatusTime`), not the agent's clock, so clock skew no longer flaps nodes UNKNOWN. +- **Status jobs** walk the fleet in small transactions and no longer delete microservices from the liveness cleaner. + +### Added + +- **Microservice last-crash columns** — `last_error`, `last_error_at`, `restart_count` on `MicroserviceStatuses` (baseline 3.9.0 and 3.8→3.9 upgrade). + +### Changed + +- **GET microservice meters** — `status.cpuUsage` and `status.memoryUsage` are 0 unless observed `status` is `RUNNING`. +- **Operator stop / application deactivate** — GET shows `STOPPING` and zero meters immediately, before the next agent PUT. +- **Microservice last crash** — GET `status` always includes `lastError`, `lastErrorAt`, and `restartCount`. `errorMessage` is the current failure (cleared by the agent after 30s RUNNING); last crash is kept after recovery. +- Embedded **EdgeOps Console** default version **v1.1.0-rc.1** → **v1.1.0-rc.2** + ## [v3.9.0-rc.2] ### Fixed diff --git a/docs/architecture.md b/docs/architecture.md index bf36dd52..e4f20716 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -131,7 +131,7 @@ Full request/response shapes: **`docs/swagger.yaml`** (agent paths). | Job | File | Role | |-----|------|------| -| Fog status | `fog-status-job.js` | Mark agents offline when status pings lapse | +| Fog status | `fog-status-job.js` | Liveness cleaner: chunked transactions mark quiet fogs UNKNOWN using Controller receipt time (`lastStatusTime`). Desired-active microservices become UNKNOWN with CPU/memory 0. Desired-inactive STOPPED workloads are left alone. Does not delete microservices. Overlapping runs on the same process are skipped. | | Controller heartbeat | `controller-heartbeat-job.js` | Control-plane liveness | | Fog token cleanup | `fog-token-cleanup-job.js` | Expire stale agent tokens | | Controller cleanup | `controller-cleanup-job.js` | Orphaned controller MS housekeeping | @@ -139,7 +139,7 @@ Full request/response shapes: **`docs/swagger.yaml`** (agent paths). | NATS reconcile | `nats-reconcile-worker-job.js` | NATS operator sync | | Platform reconcile | `platform-reconcile-worker-job.js` | Fog + service platform claim/reconcile (one job, two queues) | | Fog platform sweep | `fog-platform-sweep-job.js` | Drift detection; re-enqueue stale fog/service tasks | -| Stopped app status | `stopped-app-status-job.js` | Application state maintenance | +| Stopped app status | `stopped-app-status-job.js` | Desired-inactive safety net: leftover RUNNING, STOPPING, DELETED, or DELETING observed status is rewritten to STOPPED with CPU/memory 0. Overlapping runs on the same process are skipped. | ### Platform reconcile (fog + service + resolver) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 944ac471..e13dc478 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -8891,6 +8891,9 @@ components: type: number lastStatusTime: type: number + description: >- + Controller receipt time in milliseconds of the last successful + agent status PUT. Not the agent's clock. lastCommandTime: type: number logFileCount: @@ -9455,6 +9458,10 @@ components: type: string microserviceStatus: type: string + description: >- + JSON array of per-microservice observed status objects. Items may + include additive lastError, lastErrorAt, and restartCount. Unknown + keys must not fail this PUT. repositoryCount: type: number repositoryStatus: @@ -9463,6 +9470,9 @@ components: type: number lastStatusTime: type: number + description: >- + Agent-supplied timestamp in milliseconds. Controller liveness uses + the PUT receipt time instead of this value. ipAddress: type: string ipAddressExternal: @@ -10194,14 +10204,26 @@ components: properties: status: type: string + description: >- + Observed runtime state such as RUNNING, STOPPING, STOPPED, + UNKNOWN, or FAILED. Operator stop or application deactivate + sets STOPPING immediately, before the agent confirms STOPPED. operatingDuration: type: number + description: Live while status is RUNNING; otherwise 0. startTime: type: number + description: Live while status is RUNNING; otherwise 0. cpuUsage: type: number + description: >- + CPU sample for this microservice. Live only when status is + RUNNING (0 is valid while running). Otherwise 0. memoryUsage: type: number + description: >- + Memory sample for this microservice. Live only when status is + RUNNING (0 is valid while running). Otherwise 0. containerId: type: string podId: @@ -10209,6 +10231,29 @@ components: description: >- Pause/sandbox id on edgelet. Same as containerId on docker/podman when set. Omitted when unknown. + errorMessage: + type: string + description: >- + Current failure text. Agent keeps this populated while the + workload is failing, restarting, or has been RUNNING for less + than 30 seconds after a crash. After 30 seconds of continuous + RUNNING the agent sends an empty string so this field clears. + lastError: + type: string + description: >- + Last crash text. Not cleared on recovery. Empty string when + Controller has never stored a crash for this microservice. + lastErrorAt: + type: integer + description: >- + Unix milliseconds for lastError. 0 when lastError is empty. + restartCount: + type: integer + description: >- + Restart events since the last operator rebuild. 0 after rebuild + leftover history or when never restarted. May drop to 1 after + an agent process restart if the container is still crashed + (new episode). execStatus: type: object models: From 908748032886f2ee78c568af770c473eeaaf7a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 18 Sep 2026 20:01:05 +0300 Subject: [PATCH 31/55] Bump the embedded EdgeOps Console default to v1.1.0-rc.2. --- .env.example | 2 +- Dockerfile | 6 +++--- Makefile | 4 ++-- scripts/build-console-dev.js | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index e401c5b5..68c05529 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 +EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/Dockerfile b/Dockerfile index 8fcc3ecf..6adfb44d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM node:24-bookworm@sha256:6dac556d980b7f0e5498d08f08cee0ca67798b4ad6c23964a9214920e67758d0 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -48,9 +48,9 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds -FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:0838eaa65048d6c76749b92f423ee73af7fdd72b32b365a0fdc200b3ab74f363 +FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:d0adaa87e9c71479064b9455a1bd778a26e36ccb53be9e6bc73eae98e2be3bf6 -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index 9e323678..59083dd6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.1 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.1 +EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.2 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index 58989d29..6fa7c4ea 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.1' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.2' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { From 59ee58268a3fc1433fab97a03670c1128d0e11f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sat, 19 Sep 2026 00:28:36 +0300 Subject: [PATCH 32/55] bump edgeops console version --- .github/actions/set-build-env/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index e79ed4d2..b5906288 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.1"; fi + if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.2"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" From 6b87e69410fe8a96bbc35131b87a771afe127fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sun, 20 Sep 2026 01:30:22 +0300 Subject: [PATCH 33/55] Add VolumeMappings.scope on 3.9.0 baseline and upgrade migrations. Persist private as the default and include scope in volume mapping queries used by microservice loads and list endpoints. --- src/data/managers/microservice-manager.js | 6 +++--- src/data/managers/volume-mapping-manager.js | 2 +- .../migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql | 1 + src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql | 2 ++ .../migrations/postgres/db_migration_pg_baseline_v3.9.0.sql | 1 + src/data/migrations/postgres/db_migration_pg_v3.9.0.sql | 2 ++ .../sqlite/db_migration_sqlite_baseline_v3.9.0.sql | 1 + src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql | 2 ++ src/data/models/volumemapping.js | 6 ++++++ 9 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/data/managers/microservice-manager.js b/src/data/managers/microservice-manager.js index d643b434..72576c70 100644 --- a/src/data/managers/microservice-manager.js +++ b/src/data/managers/microservice-manager.js @@ -132,7 +132,7 @@ class MicroserviceManager extends BaseManager { model: VolumeMapping, as: 'volumeMappings', required: false, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type'] + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type', 'scope'] }, { model: CatalogItemImage, @@ -229,7 +229,7 @@ class MicroserviceManager extends BaseManager { model: VolumeMapping, as: 'volumeMappings', required: false, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type'] + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type', 'scope'] }, { model: CatalogItemImage, @@ -350,7 +350,7 @@ class MicroserviceManager extends BaseManager { model: VolumeMapping, as: 'volumeMappings', required: false, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type'] + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type', 'scope'] }, { model: CatalogItemImage, diff --git a/src/data/managers/volume-mapping-manager.js b/src/data/managers/volume-mapping-manager.js index a4c40f27..88fd68e3 100644 --- a/src/data/managers/volume-mapping-manager.js +++ b/src/data/managers/volume-mapping-manager.js @@ -10,7 +10,7 @@ class VolumeMappingManager extends BaseManager { findAll (where, transaction) { return VolumeMapping.findAll({ where, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'id', 'type'], + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'id', 'type', 'scope'], transaction }) } diff --git a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql index 2c5ab65a..45d3c144 100644 --- a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql @@ -327,6 +327,7 @@ CREATE TABLE IF NOT EXISTS VolumeMappings ( container_destination TEXT, access_mode TEXT, type TEXT, + scope VARCHAR(16) NOT NULL DEFAULT 'private', microservice_uuid VARCHAR(36), FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE ); diff --git a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql index 6084f7d6..e116a7d9 100644 --- a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql @@ -174,5 +174,7 @@ ALTER TABLE MicroserviceStatuses ADD COLUMN last_error TEXT; ALTER TABLE MicroserviceStatuses ADD COLUMN last_error_at BIGINT DEFAULT 0; ALTER TABLE MicroserviceStatuses ADD COLUMN restart_count INT DEFAULT 0; +ALTER TABLE VolumeMappings ADD COLUMN scope VARCHAR(16) NOT NULL DEFAULT 'private'; + DROP TABLE IF EXISTS HWInfos; DROP TABLE IF EXISTS USBInfos; diff --git a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql index 545030e5..fff2c7d2 100644 --- a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql @@ -325,6 +325,7 @@ CREATE TABLE IF NOT EXISTS "VolumeMappings" ( container_destination TEXT, access_mode TEXT, type TEXT, + scope VARCHAR(16) NOT NULL DEFAULT 'private', microservice_uuid VARCHAR(36), FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE ); diff --git a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql index 813b2bd2..48a6c3a7 100644 --- a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql @@ -171,5 +171,7 @@ ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS last_error TEXT DEFA ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS last_error_at BIGINT DEFAULT 0; ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS restart_count INTEGER DEFAULT 0; +ALTER TABLE "VolumeMappings" ADD COLUMN IF NOT EXISTS scope VARCHAR(16) NOT NULL DEFAULT 'private'; + DROP TABLE IF EXISTS "HWInfos"; DROP TABLE IF EXISTS "USBInfos"; diff --git a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql index f41ddc54..55a2c030 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql @@ -325,6 +325,7 @@ CREATE TABLE IF NOT EXISTS VolumeMappings ( container_destination TEXT, access_mode TEXT, type TEXT, + scope TEXT NOT NULL DEFAULT 'private', microservice_uuid VARCHAR(36), FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE ); diff --git a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql index 4eebde95..0e8c8922 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql @@ -188,5 +188,7 @@ ALTER TABLE MicroserviceStatuses ADD COLUMN last_error TEXT DEFAULT ''; ALTER TABLE MicroserviceStatuses ADD COLUMN last_error_at BIGINT DEFAULT 0; ALTER TABLE MicroserviceStatuses ADD COLUMN restart_count INTEGER DEFAULT 0; +ALTER TABLE VolumeMappings ADD COLUMN scope TEXT NOT NULL DEFAULT 'private'; + DROP TABLE IF EXISTS HWInfos; DROP TABLE IF EXISTS USBInfos; diff --git a/src/data/models/volumemapping.js b/src/data/models/volumemapping.js index a9ddce3b..4ef92d37 100644 --- a/src/data/models/volumemapping.js +++ b/src/data/models/volumemapping.js @@ -23,6 +23,12 @@ module.exports = (sequelize, DataTypes) => { type: { type: DataTypes.TEXT, field: 'type' + }, + scope: { + type: DataTypes.ENUM('private', 'shared'), + allowNull: false, + defaultValue: 'private', + field: 'scope' } }, { tableName: 'VolumeMappings', From 4117e0f5e5dc473d22f19236c460c221ccc073bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sun, 20 Sep 2026 01:30:29 +0300 Subject: [PATCH 34/55] Validate volume mapping scope and hostDestination by mapping type. Volume names use the Edgelet charset; bind mappings require an absolute host path; controller and system workloads reject explicit shared scope on type volume. --- src/helpers/volume-mapping-scope.js | 86 +++++++++++++++++++++++++++ src/schemas/microservice.js | 3 +- src/services/controller-ms-service.js | 9 ++- 3 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 src/helpers/volume-mapping-scope.js diff --git a/src/helpers/volume-mapping-scope.js b/src/helpers/volume-mapping-scope.js new file mode 100644 index 00000000..6030b64b --- /dev/null +++ b/src/helpers/volume-mapping-scope.js @@ -0,0 +1,86 @@ +'use strict' + +const path = require('path') + +const Errors = require('./errors') + +const PRIVATE_SCOPE = 'private' +const SHARED_SCOPE = 'shared' +const VOLUME_TYPE = 'volume' +const BIND_TYPE = 'bind' +const VOLUME_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/ +const WINDOWS_ABS_HOST_PATH = /^[a-zA-Z]:[\\/].+/ + +const INVALID_VOLUME_NAME_MESSAGE = 'hostDestination includes invalid characters for a local volume name, only ' + + '"[a-zA-Z0-9][a-zA-Z0-9_.-]" are allowed. If you intended to pass a host directory, use type: bind' + +const INVALID_BIND_HOST_PATH_MESSAGE = + 'hostDestination must be an absolute host path when type is bind' + +function isValidBindHostDestination (hostDestination) { + if (typeof hostDestination !== 'string') { + return false + } + const trimmed = hostDestination.trim() + if (trimmed === '') { + return false + } + return path.isAbsolute(trimmed) || WINDOWS_ABS_HOST_PATH.test(trimmed) +} + +function applyVolumeMappingScope (mapping, options = {}) { + if (!mapping) { + return mapping + } + + if (mapping.type === BIND_TYPE) { + if (!isValidBindHostDestination(mapping.hostDestination)) { + throw new Errors.InvalidArgumentError(INVALID_BIND_HOST_PATH_MESSAGE) + } + mapping.scope = PRIVATE_SCOPE + return mapping + } + + if (mapping.type !== VOLUME_TYPE) { + mapping.scope = PRIVATE_SCOPE + return mapping + } + + if (typeof mapping.hostDestination !== 'string' || !VOLUME_NAME_PATTERN.test(mapping.hostDestination)) { + throw new Errors.InvalidArgumentError(INVALID_VOLUME_NAME_MESSAGE) + } + + const raw = mapping.scope + if (raw == null) { + mapping.scope = PRIVATE_SCOPE + } else if (typeof raw !== 'string') { + throw new Errors.ValidationError(`Unknown volume mapping scope '${raw}'`) + } else { + const trimmed = raw.trim() + if (trimmed === '') { + mapping.scope = PRIVATE_SCOPE + } else { + const normalized = trimmed.toLowerCase() + if (normalized !== PRIVATE_SCOPE && normalized !== SHARED_SCOPE) { + throw new Errors.ValidationError(`Unknown volume mapping scope '${raw}'`) + } + mapping.scope = normalized + } + } + + if (options.rejectShared && mapping.scope === SHARED_SCOPE) { + throw new Errors.ValidationError( + 'Shared volume mapping scope is not allowed on controller or system microservices' + ) + } + + return mapping +} + +module.exports = { + applyVolumeMappingScope, + isValidBindHostDestination, + PRIVATE_SCOPE, + SHARED_SCOPE, + VOLUME_NAME_PATTERN +} diff --git a/src/schemas/microservice.js b/src/schemas/microservice.js index 423c6e09..56aa5a47 100644 --- a/src/schemas/microservice.js +++ b/src/schemas/microservice.js @@ -390,7 +390,8 @@ const volumeMappings = { hostDestination: { type: 'string' }, containerDestination: { type: 'string' }, accessMode: { type: 'string' }, - type: { enum: ['volume', 'bind', 'volumeMount', 'serviceAccount'] } + type: { enum: ['volume', 'bind', 'volumeMount', 'serviceAccount'] }, + scope: { type: 'string' } }, required: ['hostDestination', 'containerDestination', 'accessMode'], additionalProperties: true diff --git a/src/services/controller-ms-service.js b/src/services/controller-ms-service.js index af414d84..d4f9ef92 100644 --- a/src/services/controller-ms-service.js +++ b/src/services/controller-ms-service.js @@ -19,6 +19,7 @@ const WorkloadSpec = require('./microservice-workload-spec') const isEqual = require('lodash/isEqual') const Op = require('sequelize').Op const { VOLUME_MAPPING_DEFAULT } = require('../helpers/constants') +const { applyVolumeMappingScope, PRIVATE_SCOPE } = require('../helpers/volume-mapping-scope') const CONTROLLER_MS_NAME = 'controller' const SERVICE_ACCOUNT_VOLUME_TYPE = 'serviceAccount' @@ -78,10 +79,7 @@ function _validateVolumeMappingFields (volumeMappings) { } for (const mapping of volumeMappings) { mapping.type = mapping.type || VOLUME_MAPPING_DEFAULT - if (mapping.type === 'volume' && (!/^[a-zA-Z0-9_.-]/.test(mapping.hostDestination))) { - throw new Errors.InvalidArgumentError('hostDestination includes invalid characters for a local volume name, only ' + - '"[a-zA-Z0-9][a-zA-Z0-9_.-]" are allowed. If you intended to pass a host directory, use type: bind') - } + applyVolumeMappingScope(mapping, { rejectShared: true }) if (mapping.type === 'volumeMount') { if (!mapping.hostDestination || mapping.hostDestination === '') { throw new Errors.ValidationError('hostDestination is required when type is volumeMount') @@ -211,7 +209,8 @@ async function _createVolumeMappings (microserviceUuid, volumeMappings, transact hostDestination: volumeMapping.hostDestination, containerDestination: volumeMapping.containerDestination, accessMode: volumeMapping.accessMode, - type: volumeMapping.type || VOLUME_MAPPING_DEFAULT + type: volumeMapping.type || VOLUME_MAPPING_DEFAULT, + scope: volumeMapping.scope || PRIVATE_SCOPE })) await VolumeMappingManager.bulkCreate(mappings, transaction) } From ae65308681454628cc12658c0aa52e365401644d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sun, 20 Sep 2026 01:30:34 +0300 Subject: [PATCH 35/55] Persist volume mapping scope on create, update, and register paths. Dedicated volume-mapping create and delete set microservice rebuild and fog change tracking so agents apply mapping changes without a full microservice patch. --- src/services/microservices-service.js | 69 ++++++++++++++++++++------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/src/services/microservices-service.js b/src/services/microservices-service.js index c781c5c5..13054846 100644 --- a/src/services/microservices-service.js +++ b/src/services/microservices-service.js @@ -62,6 +62,7 @@ const Op = require('sequelize').Op const FogManager = require('../data/managers/iofog-manager') const MicroserviceExtraHostManager = require('../data/managers/microservice-extra-host-manager') const { VOLUME_MAPPING_DEFAULT } = require('../helpers/constants') +const { applyVolumeMappingScope, PRIVATE_SCOPE } = require('../helpers/volume-mapping-scope') const constants = require('../helpers/constants') const logger = require('../logger') @@ -946,15 +947,12 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) return res } -function _validateVolumeMappings (volumeMappings) { +function _validateVolumeMappings (volumeMappings, options = {}) { _stripUserServiceAccountVolumeMappings(volumeMappings) if (volumeMappings) { for (const mapping of volumeMappings) { mapping.type = mapping.type || VOLUME_MAPPING_DEFAULT - if (mapping.type === 'volume' && (!/^[a-zA-Z0-9_.-]/.test(mapping.hostDestination))) { - throw new Errors.InvalidArgumentError('hostDestination includes invalid characters for a local volume name, only ' + - '"[a-zA-Z0-9][a-zA-Z0-9_.-]" are allowed. If you intended to pass a host directory, use type: bind') - } + applyVolumeMappingScope(mapping, options) if (mapping.type === 'volumeMount') { if (!mapping.hostDestination || mapping.hostDestination === '') { throw new Errors.ValidationError('hostDestination is required when type is volumeMount') @@ -964,6 +962,14 @@ function _validateVolumeMappings (volumeMappings) { } } +async function _rejectsSharedVolumeScope (microservice, transaction) { + if (microservice.isController) { + return true + } + const app = await ApplicationManager.findOne({ id: microservice.applicationId }, transaction) + return !!(app && app.isSystem === true) +} + function _validateKeyPath (data, keyPath, resourceName, resourceType, volumeMountName) { if (!keyPath || keyPath === '') { return true // No key path to validate @@ -2249,6 +2255,13 @@ async function isMicroserviceNats (microservice, transaction) { return !!(app && app.isSystem === true) } +async function _applyVolumeMappingChangeTracking (microservice, transaction) { + await MicroserviceManager.update({ uuid: microservice.uuid }, { rebuild: true }, transaction) + if (microservice.iofogUuid) { + await _updateChangeTracking(true, microservice.iofogUuid, transaction) + } +} + async function createVolumeMappingEndPoint (microserviceUuid, volumeMappingData, isCLI, transaction) { await Validator.validate(volumeMappingData, Validator.schemas.volumeMappings) @@ -2279,7 +2292,9 @@ async function createVolumeMappingEndPoint (microserviceUuid, volumeMappingData, throw new Errors.ValidationError(ErrorMessages.VOLUME_MAPPING_ALREADY_EXISTS) } - _validateVolumeMappings([volumeMappingData]) + _validateVolumeMappings([volumeMappingData], { + rejectShared: await _rejectsSharedVolumeScope(microservice, transaction) + }) // Validate volume mount references before creating mapping // When type is 'volumeMount', validates that the volume mount exists and is linked to the fog node @@ -2292,10 +2307,13 @@ async function createVolumeMappingEndPoint (microserviceUuid, volumeMappingData, hostDestination: volumeMappingData.hostDestination, containerDestination: volumeMappingData.containerDestination, accessMode: volumeMappingData.accessMode, - type + type: volumeMappingData.type || type, + scope: volumeMappingData.scope || PRIVATE_SCOPE } - return VolumeMappingManager.create(volumeMappingObj, transaction) + const created = await VolumeMappingManager.create(volumeMappingObj, transaction) + await _applyVolumeMappingChangeTracking(microservice, transaction) + return created } async function createSystemVolumeMappingEndPoint (microserviceUuid, volumeMappingData, isCLI, transaction) { @@ -2328,7 +2346,9 @@ async function createSystemVolumeMappingEndPoint (microserviceUuid, volumeMappin throw new Errors.ValidationError(ErrorMessages.VOLUME_MAPPING_ALREADY_EXISTS) } - _validateVolumeMappings([volumeMappingData]) + _validateVolumeMappings([volumeMappingData], { + rejectShared: await _rejectsSharedVolumeScope(microservice, transaction) + }) // Validate volume mount references before creating mapping // When type is 'volumeMount', validates that the volume mount exists and is linked to the fog node @@ -2341,10 +2361,13 @@ async function createSystemVolumeMappingEndPoint (microserviceUuid, volumeMappin hostDestination: volumeMappingData.hostDestination, containerDestination: volumeMappingData.containerDestination, accessMode: volumeMappingData.accessMode, - type + type: volumeMappingData.type || type, + scope: volumeMappingData.scope || PRIVATE_SCOPE } - return VolumeMappingManager.create(volumeMappingObj, transaction) + const created = await VolumeMappingManager.create(volumeMappingObj, transaction) + await _applyVolumeMappingChangeTracking(microservice, transaction) + return created } async function deleteVolumeMappingEndPoint (microserviceUuid, volumeMappingUuid, isCLI, transaction) { @@ -2377,6 +2400,8 @@ async function deleteVolumeMappingEndPoint (microserviceUuid, volumeMappingUuid, if (affectedRows === 0) { throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_VOLUME_MAPPING_UUID, volumeMappingUuid)) } + + await _applyVolumeMappingChangeTracking(microservice, transaction) } async function deleteSystemVolumeMappingEndPoint (microserviceUuid, volumeMappingUuid, isCLI, transaction) { @@ -2409,6 +2434,8 @@ async function deleteSystemVolumeMappingEndPoint (microserviceUuid, volumeMappin if (affectedRows === 0) { throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_VOLUME_MAPPING_UUID, volumeMappingUuid)) } + + await _applyVolumeMappingChangeTracking(microservice, transaction) } async function listVolumeMappingsEndPoint (microserviceUuid, isCLI, transaction) { @@ -2643,23 +2670,30 @@ async function _createVolumeMappings (microservice, volumeMappings, transaction) const mappings = [] for (const volumeMapping of volumeMappings) { - const mapping = Object.assign({}, volumeMapping) - mapping.microserviceUuid = microservice.uuid - mappings.push(mapping) + mappings.push({ + microserviceUuid: microservice.uuid, + hostDestination: volumeMapping.hostDestination, + containerDestination: volumeMapping.containerDestination, + accessMode: volumeMapping.accessMode, + type: volumeMapping.type || VOLUME_MAPPING_DEFAULT, + scope: volumeMapping.scope || PRIVATE_SCOPE + }) } await VolumeMappingManager.bulkCreate(mappings, transaction) } async function _updateVolumeMappings (volumeMappings, microserviceUuid, transaction) { - _validateVolumeMappings(volumeMappings) - // Get microservice to find fogUuid for volume mount validation const microservice = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) if (!microservice) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, microserviceUuid)) } + _validateVolumeMappings(volumeMappings, { + rejectShared: await _rejectsSharedVolumeScope(microservice, transaction) + }) + // Validate volume mount references before updating mappings // When type is 'volumeMount', validates that the volume mount exists and is linked to the fog node if (volumeMappings && microservice.iofogUuid) { @@ -2682,7 +2716,8 @@ async function _updateVolumeMappings (volumeMappings, microserviceUuid, transact hostDestination: volumeMapping.hostDestination, containerDestination: volumeMapping.containerDestination, accessMode: volumeMapping.accessMode, - type + type, + scope: volumeMapping.scope || PRIVATE_SCOPE } await VolumeMappingManager.create(volumeMappingObj, transaction) From 22c2ea251df176f00dadb705c10ca6dcaa8648ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sun, 20 Sep 2026 01:30:39 +0300 Subject: [PATCH 36/55] Add tests for volume mapping scope, bind path rules, and agent list output. Cover create and update persistence, controller register, YAML round-trip, and dedicated volume-mapping routes including rebuild side effects. --- test/src/helpers/volume-mapping-scope.test.js | 119 ++++ test/src/services/agent-service.test.js | 18 + .../services/controller-ms-service.test.js | 21 + .../services/microservices-service.test.js | 560 ++++++++++++++++++ test/src/services/yaml-parser-service.test.js | 25 + 5 files changed, 743 insertions(+) create mode 100644 test/src/helpers/volume-mapping-scope.test.js diff --git a/test/src/helpers/volume-mapping-scope.test.js b/test/src/helpers/volume-mapping-scope.test.js new file mode 100644 index 00000000..dedadb89 --- /dev/null +++ b/test/src/helpers/volume-mapping-scope.test.js @@ -0,0 +1,119 @@ +'use strict' + +const { expect } = require('chai') + +const { + applyVolumeMappingScope, + isValidBindHostDestination +} = require('../../../src/helpers/volume-mapping-scope') +const Errors = require('../../../src/helpers/errors') + +function volumeMapping (overrides = {}) { + return { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + ...overrides + } +} + +describe('volume mapping scope helper', () => { + describe('type volume', () => { + it('stores private when scope is omitted', () => { + const mapping = volumeMapping() + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + }) + + it('stores private when scope is empty, null, or whitespace', () => { + for (const scope of ['', null, ' ']) { + const mapping = volumeMapping({ scope }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + } + }) + + it('stores lowercase shared when scope is Shared', () => { + const mapping = volumeMapping({ scope: 'Shared' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('shared') + }) + + it('rejects unknown scope values', () => { + expect(() => applyVolumeMappingScope(volumeMapping({ scope: 'Shareed' }))) + .to.throw(Errors.ValidationError, /Unknown volume mapping scope/) + }) + + it('accepts nodered-config as a volume name', () => { + const mapping = volumeMapping({ hostDestination: 'nodered-config' }) + applyVolumeMappingScope(mapping) + expect(mapping.hostDestination).to.equal('nodered-config') + expect(mapping.scope).to.equal('private') + }) + + it('rejects volume names that start with a dot, underscore, or path traversal', () => { + for (const hostDestination of ['../x', '.hidden', '_scratch']) { + expect(() => applyVolumeMappingScope(volumeMapping({ hostDestination }))) + .to.throw(Errors.InvalidArgumentError, /invalid characters/) + } + }) + + it('persists shared with read-only access', () => { + const mapping = volumeMapping({ scope: 'shared', accessMode: 'ro' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('shared') + expect(mapping.accessMode).to.equal('ro') + }) + + it('rejects explicit shared when the microservice is controller or system', () => { + expect(() => applyVolumeMappingScope(volumeMapping({ scope: 'shared' }), { rejectShared: true })) + .to.throw(Errors.ValidationError, /not allowed on controller or system/) + }) + + it('stores private on controller or system when shared is not sent', () => { + const mapping = volumeMapping() + applyVolumeMappingScope(mapping, { rejectShared: true }) + expect(mapping.scope).to.equal('private') + }) + }) + + describe('type bind', () => { + it('accepts absolute unix and windows host paths', () => { + for (const hostDestination of ['/var/lib/data', 'C:\\data\\app']) { + expect(isValidBindHostDestination(hostDestination)).to.equal(true) + const mapping = volumeMapping({ + type: 'bind', + hostDestination, + scope: 'shared' + }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + } + }) + + it('rejects relative host paths and volume-style names on bind', () => { + for (const hostDestination of ['relative/path', 'nodered-config', '../x']) { + expect(isValidBindHostDestination(hostDestination)).to.equal(false) + expect(() => applyVolumeMappingScope(volumeMapping({ type: 'bind', hostDestination }))) + .to.throw(Errors.InvalidArgumentError, /absolute host path/) + } + }) + }) + + describe('non-volume types', () => { + it('stores private when type is omitted and scope is shared', () => { + const mapping = volumeMapping({ type: undefined, scope: 'shared' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + }) + + it('stores private and does not reject shared on volumeMount or serviceAccount', () => { + for (const type of ['volumeMount', 'serviceAccount']) { + const mapping = volumeMapping({ type, scope: 'shared' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + } + }) + }) +}) diff --git a/test/src/services/agent-service.test.js b/test/src/services/agent-service.test.js index a1feec4d..89e75ee7 100644 --- a/test/src/services/agent-service.test.js +++ b/test/src/services/agent-service.test.js @@ -1398,6 +1398,24 @@ describe('Agent Service', () => { expect(msvc.isNats).to.equal(false) }) + it('includes volume mapping scope from the persisted mapping', async () => { + const volumeMappings = [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + MicroserviceManager.findAllActiveApplicationMicroservices.resolves([{ + ...microserviceWithValidImage, + volumeMappings + }]) + + const result = await $subject + expect(result.microservices[0].volumeMappings).to.eql(volumeMappings) + expect(result.microservices[0].volumeMappings[0].scope).to.equal('shared') + }) + it('emits catalog models and omits empty argv arrays', async () => { MicroserviceManager.findAllActiveApplicationMicroservices.resolves([{ ...microserviceWithValidImage, diff --git a/test/src/services/controller-ms-service.test.js b/test/src/services/controller-ms-service.test.js index c5841bc7..72c127a9 100644 --- a/test/src/services/controller-ms-service.test.js +++ b/test/src/services/controller-ms-service.test.js @@ -166,6 +166,27 @@ describe('Controller MS Service', () => { expect(MicroservicesService.createOrUpdateServiceAccountForMicroservice).to.not.have.been.called }) + context('when register sends a shared volume mapping', () => { + def('body', () => ({ + ...registerData, + volumeMappings: [{ + hostDestination: 'db', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + })) + + it('rejects shared scope on controller volumes', async () => { + await expect($subject).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + expect(VolumeMappingManager.bulkCreate).to.not.have.been.called + }) + }) + context('when container workload fields are sent on create', () => { def('body', () => ({ ...registerData, diff --git a/test/src/services/microservices-service.test.js b/test/src/services/microservices-service.test.js index ed2ed71b..ec174d16 100644 --- a/test/src/services/microservices-service.test.js +++ b/test/src/services/microservices-service.test.js @@ -405,6 +405,176 @@ describe('Microservices Service', () => { }) }) + context('when volumeMappings include scope', () => { + const basePayload = { + name: 'new-msvc', + application: 'my-app', + iofogUuid: 'fog-uuid', + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1 + } + + it('stores private when type is volume and scope is omitted', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0].scope).to.equal('private') + }) + + it('stores private when type is volume and scope is empty or null', async () => { + for (const scope of ['', null]) { + VolumeMappingManager.bulkCreate.resetHistory() + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0].scope).to.equal('private') + } + }) + + it('stores shared when type is volume and scope is Shared', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ + hostDestination: 'nodered-config', + type: 'volume', + scope: 'shared' + }) + }) + + it('rejects unknown scope on type volume', async () => { + await expect($service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shareed' + }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /Unknown volume mapping scope/) + }) + + it('stores private when type is omitted and scope is shared', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: '/host/data', + containerDestination: '/data', + accessMode: 'rw', + scope: 'shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ type: 'bind', scope: 'private' }) + }) + + it('stores private when type is bind and scope is unknown', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: '/host/data', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind', + scope: 'nope' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ type: 'bind', scope: 'private' }) + }) + + it('persists both a private and a shared mapping with the same volume name', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'config', + containerDestination: '/private-config', + accessMode: 'rw', + type: 'volume' + }, { + hostDestination: 'config', + containerDestination: '/shared-config', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings).to.have.length(2) + expect(mappings[0]).to.include({ + hostDestination: 'config', + containerDestination: '/private-config', + scope: 'private' + }) + expect(mappings[1]).to.include({ + hostDestination: 'config', + containerDestination: '/shared-config', + scope: 'shared' + }) + }) + + it('persists shared with read-only access', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'ro', + type: 'volume', + scope: 'shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ + accessMode: 'ro', + scope: 'shared' + }) + }) + + it('rejects a volume name that is not a valid local volume name', async () => { + await expect($service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: '../x', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume' + }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.InvalidArgumentError, /invalid characters/) + }) + }) + context('when runtime is set', () => { it('auto-links the runtime class on an edgelet fog', async () => { FogManager.findOne.resolves({ @@ -668,6 +838,41 @@ describe('Microservices Service', () => { }) }) }) + + context('when volumeMappings change scope', () => { + def('updateData', () => ({ + volumeMappings: [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + })) + + it('marks the microservice for rebuild and notifies the node', async () => { + await $subject + expect(MicroserviceManager.updateAndFind).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + microserviceUuid: msvcUuid, + hostDestination: 'nodered-config', + type: 'volume', + scope: 'shared' + }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + }) }) describe('.updateSystemMicroserviceEndPoint()', () => { @@ -709,6 +914,25 @@ describe('Microservices Service', () => { ) }) + context('when a controller microservice is sent a shared volume', () => { + def('updateData', () => ({ + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + })) + + it('rejects shared scope on a controller microservice', () => { + return expect($subject).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + }) + context('when microservice is not controller', () => { def('updateData', () => ({ schedule: 4 })) @@ -864,8 +1088,11 @@ describe('Microservices Service', () => { beforeEach(() => { $sandbox.stub(Validator, 'validate').resolves(true) $sandbox.stub(MicroserviceManager, 'findMicroserviceOnGet').resolves(microservice) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ id: microservice.applicationId, isSystem: false }) $sandbox.stub(VolumeMappingManager, 'findOne').resolves(null) $sandbox.stub(VolumeMappingManager, 'create').resolves({ uuid: 'vol-uuid' }) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() }) it('creates a user volume mapping', async () => { @@ -873,6 +1100,175 @@ describe('Microservices Service', () => { expect(VolumeMappingManager.create).to.have.been.calledOnce }) + it('persists default private scope and marks the microservice for rebuild', async () => { + await $subject + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + microserviceUuid: msvcUuid, + hostDestination: 'data', + type: 'volume', + scope: 'private' + }), + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('persists shared scope on type volume', async () => { + await $service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shared' + }, isCLI, transaction) + + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + hostDestination: 'nodered-config', + type: 'volume', + scope: 'shared' + }), + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('rejects bind when hostDestination is not an absolute path', () => { + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind' + }, isCLI, transaction)).to.be.rejectedWith(Errors.InvalidArgumentError, /absolute host path/) + }) + + it('stores private when type is bind and scope is shared', async () => { + await $service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: '/host/data', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind', + scope: 'shared' + }, isCLI, transaction) + + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + type: 'bind', + scope: 'private' + }), + transaction + ) + }) + + it('rejects unknown scope on type volume', () => { + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shareed' + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /Unknown volume mapping scope/) + }) + + it('rejects shared scope on a system microservice', () => { + ApplicationManager.findOne.resolves({ id: microservice.applicationId, isSystem: true }) + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }, isCLI, transaction)).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + + it('rejects shared scope on a controller microservice', () => { + MicroserviceManager.findMicroserviceOnGet.resolves(buildMicroserviceRecord({ + uuid: msvcUuid, + isController: true + })) + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }, isCLI, transaction)).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + + it('allows the same shared volume name on two user microservices on one node', async () => { + const otherMs = buildMicroserviceRecord({ uuid: 'other-msvc', iofogUuid: 'fog-uuid' }) + MicroserviceManager.findMicroserviceOnGet.callsFake((where) => { + if (where && where.uuid === 'other-msvc') { + return Promise.resolve(otherMs) + } + return Promise.resolve(microservice) + }) + const mapping = { + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + } + + await $service.createVolumeMappingEndPoint(msvcUuid, mapping, isCLI, transaction) + await $service.createVolumeMappingEndPoint('other-msvc', { ...mapping }, isCLI, transaction) + + expect(VolumeMappingManager.create).to.have.been.calledTwice + expect(VolumeMappingManager.create.firstCall.args[0]).to.include({ + microserviceUuid: msvcUuid, + hostDestination: 'nodered-config', + scope: 'shared' + }) + expect(VolumeMappingManager.create.secondCall.args[0]).to.include({ + microserviceUuid: 'other-msvc', + hostDestination: 'nodered-config', + scope: 'shared' + }) + }) + + it('skips fog change tracking when the microservice has no node', async () => { + MicroserviceManager.findMicroserviceOnGet.resolves(buildMicroserviceRecord({ + uuid: msvcUuid, + iofogUuid: null + })) + + await $subject + + expect(VolumeMappingManager.create).to.have.been.calledOnce + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.called + }) + context('when type is serviceAccount', () => { def('volumeMappingData', () => ({ hostDestination: 'immutable-name', @@ -889,6 +1285,170 @@ describe('Microservices Service', () => { }) }) + describe('.createSystemVolumeMappingEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + + def('subject', () => $service.createSystemVolumeMappingEndPoint(msvcUuid, $volumeMappingData, isCLI, transaction)) + def('volumeMappingData', () => ({ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume' + })) + + beforeEach(() => { + $sandbox.stub(Validator, 'validate').resolves(true) + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ id: microservice.applicationId, isSystem: true }) + $sandbox.stub(VolumeMappingManager, 'findOne').resolves(null) + $sandbox.stub(VolumeMappingManager, 'create').resolves({ uuid: 'vol-uuid' }) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('persists private scope and marks the system microservice for rebuild', async () => { + await $subject + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + microserviceUuid: msvcUuid, + type: 'volume', + scope: 'private' + }), + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('rejects explicit shared scope on a system microservice volume', () => { + return expect($service.createSystemVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }, isCLI, transaction)).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + }) + + describe('.deleteVolumeMappingEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const mappingUuid = 'vol-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + + def('subject', () => $service.deleteVolumeMappingEndPoint(msvcUuid, mappingUuid, isCLI, transaction)) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(VolumeMappingManager, 'findOne').resolves({ + uuid: mappingUuid, + microserviceUuid: msvcUuid, + type: 'volume' + }) + $sandbox.stub(VolumeMappingManager, 'delete').resolves(1) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes the mapping and marks the microservice for rebuild', async () => { + await $subject + expect(VolumeMappingManager.delete).to.have.been.calledWith( + { uuid: mappingUuid, microserviceUuid: msvcUuid }, + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('rejects delete of system-managed serviceAccount mappings', () => { + VolumeMappingManager.findOne.resolves({ + uuid: mappingUuid, + microserviceUuid: msvcUuid, + type: 'serviceAccount' + }) + return expect($subject).to.be.rejectedWith( + 'Volume mappings of type serviceAccount are system-managed and cannot be deleted by users' + ) + }) + }) + + describe('.deleteSystemVolumeMappingEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const mappingUuid = 'vol-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(VolumeMappingManager, 'findOne').resolves({ + uuid: mappingUuid, + microserviceUuid: msvcUuid, + type: 'volume' + }) + $sandbox.stub(VolumeMappingManager, 'delete').resolves(1) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes the mapping and marks the system microservice for rebuild', async () => { + await $service.deleteSystemVolumeMappingEndPoint(msvcUuid, mappingUuid, isCLI, transaction) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + }) + + describe('.listVolumeMappingsEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + const mappings = [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + id: 7, + type: 'volume', + scope: 'shared' + }] + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(VolumeMappingManager, 'findAll').resolves(mappings) + }) + + it('returns stored mapping scope', async () => { + const result = await $service.listVolumeMappingsEndPoint(msvcUuid, isCLI, transaction) + expect(VolumeMappingManager.findAll).to.have.been.calledWith({ microserviceUuid: msvcUuid }, transaction) + expect(result).to.eql(mappings) + expect(result[0].scope).to.equal('shared') + }) + }) + describe('.createMicroserviceEndPoint() catalog and registry', () => { const msvcUuid = 'new-msvc-uuid' const microserviceData = { diff --git a/test/src/services/yaml-parser-service.test.js b/test/src/services/yaml-parser-service.test.js index dcfde491..2e03294c 100644 --- a/test/src/services/yaml-parser-service.test.js +++ b/test/src/services/yaml-parser-service.test.js @@ -210,6 +210,31 @@ spec: expect(result).to.not.have.property('volumes') }) + it('passes container volume scope through to volumeMappings', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-shared-volume +spec: + container: + volumes: + - hostDestination: nodered-config + containerDestination: /data + accessMode: rw + type: volume + scope: shared + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.volumeMappings).to.eql([{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }]) + }) + it('does not map deprecated top-level natsAccess', async () => { const yaml = ` kind: Microservice From 784a943da968396d1ff786e936ae9891bbdf8bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sun, 20 Sep 2026 01:30:54 +0300 Subject: [PATCH 37/55] Document volume mapping scope for v3.9.0-rc.4 in swagger and release notes. Describe private versus shared scope, hostDestination rules for volume versus bind, and agent-visible lowercase scope on each mapping. --- CHANGELOG.md | 23 +++++++++++++++++++++++ docs/swagger.yaml | 28 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc22d333..ebb60b78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [v3.9.0-rc.4] + +Additive microservice `VOLUME` mapping `scope` for Edgelet private vs node-global shared claims. Reclaim stays on-node (`edgelet volume`); no new Controller prune flag. + +### Added + +- **`volumeMappings[].scope`** — `private` (default) or `shared`. Meaningful only when `type` is `volume`. Omit, null, or empty → `private`. Unknown values on `type: volume` are **400**. Other mapping types always store `private`. +- **`VolumeMappings.scope`** column on 3.9.0 baseline and 3.8→3.9 upgrade (sqlite, mysql, postgres). + +### Changed + +- **Agent microservice list** includes `scope` on each volume mapping (lowercase). +- **Volume-mapping create/delete** sets microservice `rebuild` and `microserviceCommon` so the agent applies the change. +- **`type: volume` `hostDestination`** charset matches Edgelet: `[a-zA-Z0-9][a-zA-Z0-9_.-]*`. +- Embedded **EdgeOps Console** default version **v1.1.0-rc.2** → **v1.1.0-rc.3** + +### Notes + +- Shared names are node-global. Controller does not unique-check names or `runAsUser` across consumers (local workloads exist). Use distinct names (`nodered-config`). +- Controller and system microservices always persist `private`. Explicit `scope: shared` on those workloads is **400**. +- `deleteWithCleanup` remains unused; operators reclaim with `edgelet volume` / `deprovision --purge-volumes`. +- Switching private → shared does not copy existing UUID directories. Older agents ignore `scope` (every `VOLUME` stays per-UUID) with no merge after upgrade. + ## [v3.9.0-rc.3] Observed microservice status and fog liveness: live CPU/memory only while a microservice is RUNNING, and node quietness judged by when Controller last received a status PUT. diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e13dc478..8aca045e 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -9807,6 +9807,18 @@ components: accessMode: type: string example: rw + scope: + type: string + enum: + - private + - shared + default: private + description: >- + Meaningful only when type is volume. Omit, null, or empty stores + private. Unknown values on type volume return 400. Other mapping + types always store private. Runtime accepts any case and stores + lowercase. + example: private VolumeMappingResponse: type: object properties: @@ -9821,6 +9833,10 @@ components: type: number hostDestination: type: string + description: >- + When type is volume, a managed volume name + ([a-zA-Z0-9][a-zA-Z0-9_.-]*). When type is bind (default), an + absolute host path (Unix or Windows drive path). example: /var/dest containerDestination: type: string @@ -9831,6 +9847,18 @@ components: type: type: string example: bind + scope: + type: string + enum: + - private + - shared + default: private + description: >- + Meaningful only when type is volume. Omit, null, or empty stores + private. Unknown values on type volume return 400. Other mapping + types always store private. Runtime accepts any case and stores + lowercase. GET always returns private or shared in lowercase. + example: private PortMappingsResponse: type: object properties: From 88ace0efcfec1ab2e79a3d07d31ec5e13a56fab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sun, 20 Sep 2026 01:31:08 +0300 Subject: [PATCH 38/55] Bump the embedded EdgeOps Console default to v1.1.0-rc.3. Align local build defaults, Docker build-args, and development env example with the rc.3 console embed. --- .env.example | 2 +- .github/actions/set-build-env/action.yml | 2 +- Dockerfile | 4 ++-- Makefile | 4 ++-- README.md | 4 +++- scripts/build-console-dev.js | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 68c05529..e7c45fd8 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 +EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index b5906288..b71f5c90 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.2"; fi + if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.3"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/Dockerfile b/Dockerfile index 6adfb44d..14733d94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM node:24-bookworm@sha256:6dac556d980b7f0e5498d08f08cee0ca67798b4ad6c23964a9214920e67758d0 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -50,7 +50,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:d0adaa87e9c71479064b9455a1bd778a26e36ccb53be9e6bc73eae98e2be3bf6 -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index 59083dd6..63ae0040 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.2 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.2 +EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.3 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/README.md b/README.md index 513dde28..74280ccf 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,9 @@ API: `http://localhost:51121` · Console (embedded or split): set `CONSOLE_URL` |-------|-----| | Architecture | [docs/architecture.md](docs/architecture.md) | | RBAC | [docs/rbac-reference.md](docs/rbac-reference.md) | -| External OIDC | [docs/external-oidc-client-setup.md](docs/external-oidc-client-setup.md) | +| OIDC (embedded and external) | [docs/oidc-configuration.md](docs/oidc-configuration.md) | +| External OIDC client | [docs/external-oidc-client-setup.md](docs/external-oidc-client-setup.md) | +| External OIDC providers | [docs/external-oidc-providers.md](docs/external-oidc-providers.md) | ## License diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index 6fa7c4ea..a8188d49 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.2' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.3' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { From 4b6d8bf8b9c6a3f6a2e38b9528e05c7e8fe83ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Sun, 20 Sep 2026 01:31:27 +0300 Subject: [PATCH 39/55] Expand external OIDC client setup and add provider-specific guidance. Clarify BFF versus CLI login, agent route exceptions, and link operator docs for IdP recipes and RBAC mapping. --- docs/architecture.md | 5 +- docs/external-oidc-client-setup.md | 149 +++++++++------- docs/external-oidc-providers.md | 269 +++++++++++++++++++++++++++++ docs/oidc-configuration.md | 5 +- docs/rbac-reference.md | 5 +- 5 files changed, 369 insertions(+), 64 deletions(-) create mode 100644 docs/external-oidc-providers.md diff --git a/docs/architecture.md b/docs/architecture.md index e4f20716..ba6edaa6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -109,7 +109,7 @@ Used by EdgeOps Console, `potctl`, `iofogctl`, and automation. | Certificates | `/api/v3/certificate` | PKI operations | | WebSocket | `ws` routes in `src/routes/` | Exec and logs (Bearer token) | -Browser login uses the OAuth BFF (`GET /api/v3/user/oauth/authorize`); CLI uses `POST /api/v3/user/login`. See [oidc-configuration.md](oidc-configuration.md), [external-oidc-client-setup.md](external-oidc-client-setup.md), and [rbac-reference.md](rbac-reference.md). +Browser login uses the OAuth BFF (`GET /api/v3/user/oauth/authorize`); CLI uses `POST /api/v3/user/login`. See [oidc-configuration.md](oidc-configuration.md), [external-oidc-client-setup.md](external-oidc-client-setup.md), [external-oidc-providers.md](external-oidc-providers.md), and [rbac-reference.md](rbac-reference.md). ### Agent API (fog token — no OIDC) @@ -356,6 +356,8 @@ Controller v3.8 and Edgelet share a **frozen field-agent REST contract** on `/ap **Volume mounts** — `GET volumeMounts` returns bind/volume shapes plus system-injected immutable `serviceAccount` entries. +**Volume mapping scope** — microservice `volumeMappings[]` may include `scope` (`private` default / `shared`). Meaningful only when `type` is `volume`. Shared names are node-global on that fog. Reclaim is on-node (`edgelet volume` / `deprovision --purge-volumes`); Controller does not prune volumes. `deleteWithCleanup` remains unused. + **Service account RBAC** — microservice roles use apiGroup **`edgelet.iofog.org/v1`**; SA/role changes trigger `microserviceList` change tracking. For the full bilateral contract (including ControlPlane env vars and verification references), see Edgelet documentation: @@ -466,6 +468,7 @@ Agent routes and WebSocket exec/logs for agents are **outside** OIDC — see [rb | [pki.md](pki.md) | Central CAs, cert renewal, NATS operator rotation | | [oidc-configuration.md](oidc-configuration.md) | Embedded/external auth modes and env vars | | [external-oidc-client-setup.md](external-oidc-client-setup.md) | External IdP client configuration | +| [external-oidc-providers.md](external-oidc-providers.md) | Default groups/roles and IdP recipes | | [operations/database-transactions.md](operations/database-transactions.md) | Transaction runner, OTEL metrics, SQLITE_BUSY runbook | | [operations/sizing.md](operations/sizing.md) | Hardware sizing by fog count (K8s and Remote CP) | | [CONTRIBUTING](../CONTRIBUTING) | Dual-mirror CI and development | diff --git a/docs/external-oidc-client-setup.md b/docs/external-oidc-client-setup.md index 4e75ea4d..777e86d8 100644 --- a/docs/external-oidc-client-setup.md +++ b/docs/external-oidc-client-setup.md @@ -1,12 +1,14 @@ -# External OIDC provider — minimum client setup for Controller +# External OIDC provider — client setup for Controller **Audience:** Platform and IdP administrators **Controller mode:** `AUTH_MODE=external` -**Applies to:** Any OIDC-compliant provider (Keycloak, Auth0, Okta, Azure AD, etc.) +**Applies to:** Any OIDC-compliant provider (Keycloak, Microsoft Entra ID, Okta, Auth0, and similar) + +This page is the **OIDC client contract**: client type, grants, redirect URIs, scopes, and access-token rules. For default groups/roles and per-provider recipes, see [external-oidc-providers.md](external-oidc-providers.md). Operator env and login flows: [oidc-configuration.md](oidc-configuration.md). ## Overview -Controller uses one **confidential** OIDC client for browser and CLI authentication when `AUTH_MODE=external`. +Controller uses **one confidential** OIDC client for browser and CLI authentication when `AUTH_MODE=external`. The EdgeOps Console does not talk to the IdP as a public SPA. Controller is an OAuth BFF for the browser and a password-grant front door for the CLI. | Use case | Grant / flow | Controller endpoint | |----------|--------------|---------------------| @@ -14,37 +16,67 @@ Controller uses one **confidential** OIDC client for browser and CLI authenticat | CLI (potctl) | Resource owner password (direct access) | `POST /api/v3/user/login` | | Session refresh | Refresh token | `POST /api/v3/user/refresh` | | Profile | Bearer access token (+ UserInfo) | `GET /api/v3/user/profile` | +| User APIs and operator WebSockets | Bearer access JWT | `/api/v3/*` except agent routes | +| Agent routes | Fog token | `/api/v3/agent/*` — OIDC does **not** apply | + +In external mode: -In external mode, access and refresh tokens are **issued by the IdP**. Controller validates access JWTs via the issuer JWKS and maps claims to RBAC. +- Access and refresh tokens are **issued by the IdP**. Controller does not mint them. +- Controller validates **access** JWTs via the issuer JWKS (`aud` must be `OIDC_CLIENT_ID`). +- There is **no local `AuthUsers` row** for IdP users. RBAC comes from JWT claims (and optional RoleBindings). +- MFA, password policy, and forced password change are **owned by the IdP**. Controller does not run embedded interaction UI in this mode. ## Controller environment (minimum) | Variable | Required | Example | |----------|----------|---------| | `AUTH_MODE` | Yes | `external` | -| `OIDC_ISSUER_URL` | Yes | `https://auth.example.com/realms/myrealm` | +| `OIDC_ISSUER_URL` | Yes | Full issuer URL (see [providers](external-oidc-providers.md)) | | `OIDC_CLIENT_ID` | Yes | `pot-controller` | | `OIDC_CLIENT_SECRET` | Yes | Confidential client secret | | `CONTROLLER_PUBLIC_URL` | Yes | `https://controller.example.com` | | `CONSOLE_URL` | Yes (browser login) | `https://console.example.com` | +| `TRUST_PROXY` | When TLS terminates at ingress | `true` | +| `AUTH_SESSION_STORE_TYPE` | HA + browser login | `database` (mysql/postgres) | | `AUTH_INSECURE_ALLOW_HTTP` | Development only | `true` when using `http://localhost:*` | +`OIDC_ISSUER_URL` must be the **issuer string**, not a generic IdP homepage. Discovery is: + +```text +GET {OIDC_ISSUER_URL}/.well-known/openid-configuration +``` + +Do **not** set legacy `KC_*`, `auth.realm`, or `auth.realmKey`. + +Optional `OIDC_CONSOLE_CLIENT_ID` / `AUTH_CONSOLE_CLIENT_ENABLED` registers a **future SPA-direct** public client. Default is **off**. Primary Console login uses this confidential Controller client via the BFF. Do not point Console at a second public client unless you intend to enable that path. + ## IdP client — required settings ### Client type -- OpenID Connect -- **Confidential client** (client authentication enabled; uses `OIDC_CLIENT_SECRET`) -- Public clients are not supported for the Controller OAuth BFF +Create **one** client for Controller. Do not use a public SPA, native, or device client as this client. + +| Setting | Required value | Why | +|---------|----------------|-----| +| Protocol | OpenID Connect | Controller is OIDC-only | +| Client type | **Confidential** (web application) | BFF uses `OIDC_CLIENT_SECRET` at the token endpoint | +| Client authentication | `client_secret_basic` or `client_secret_post` | Public (`none`) is not supported | +| PKCE | **S256 required or allowed** | Browser authorize always sends `code_challenge_method=S256` | +| Implicit / hybrid | **Off** | Not used | +| Device code | Off | Not used | +| Client credentials grant | Unused | Not used for operator login | ### Enabled grant types / flows | Flow | Required for | Notes | |------|--------------|-------| | Standard flow (authorization code) | Browser Sign in | Required | +| Refresh token (`offline_access`) | Console/CLI session refresh | Required for `POST /api/v3/user/refresh` | | Direct access grants (ROPC) | CLI `POST /user/login` | Required if potctl uses password login against the IdP | | Implicit flow | — | Off (not used) | +If your organization has disabled ROPC (common on Entra ID), browser login still works; CLI password login will not. + ### PKCE Controller always sends PKCE S256 on browser authorize (`code_challenge`, `code_challenge_method=S256`). @@ -58,7 +90,7 @@ Disabling PKCE on the IdP is a development workaround only, not recommended for ### Redirect URIs -Register exactly: +Register exactly as a **Web** redirect URI (not SPA): ```text {CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback @@ -73,19 +105,19 @@ Avoid overly broad wildcards in production. ### Web origins (CORS) -If the Console calls the Controller API from the browser, allow: +If the Console calls the Controller API from a different origin, allow: ```text {CONSOLE_URL} ``` -Example: `http://localhost:3000` +Example: `http://localhost:3000` or `https://console.example.com` ## Scopes ### Requested by Controller (browser OAuth BFF) -Controller sends this scope string on authorize: +Controller sends this scope string on authorize (not configurable): ```text openid profile email groups offline_access @@ -96,7 +128,7 @@ openid profile email groups offline_access | `openid` | OIDC baseline; `sub`, `id_token` | | `profile` | `preferred_username`, display name | | `email` | Email claim; identity linking | -| `groups` | RBAC group membership (see below) | +| `groups` | RBAC group membership — name must be **`groups`**, not `group` | | `offline_access` | Refresh token on authorization code flow | ### IdP client scope assignment @@ -106,36 +138,24 @@ Every requested scope must be assigned to the client as a **default** and/or **o | Scope | Typical assignment | |-------|-------------------| | `openid`, `profile`, `email`, `roles` | Default | -| `groups` | Optional (scope name must be **`groups`**, not `group`) | +| `groups` | Optional (scope name must be **`groups`**) | | `offline_access` | Optional | -**Common error:** `invalid_scope` when a scope is requested but not assigned to the client, or when the scope name is wrong (`group` vs `groups`). - -## RBAC / groups mapping +**Common error:** `invalid_scope` when a scope is requested but not assigned to the client, when the scope name is wrong (`group` vs `groups`), or when the provider only accepts a full URI scope (`api://…/groups`) instead of the short name `groups`. -Controller resolves RBAC **Group** subjects from the access token (in order): +## Access token requirements -1. `resource_access[{OIDC_CLIENT_ID}].roles` — Keycloak client roles (recommended) -2. Top-level `roles` array (if present) -3. `groups` array — from the `groups` scope and a group mapper +Controller validates the **access token**, not the ID token, on API calls. Groups and roles on the ID token alone are ignored for authorization. -Group names are lowercased before RBAC lookup. IdP role and group names should align with Controller RBAC groups (for example `admin`, `viewer`). +The access token **must**: -### Keycloak options +1. Be a **JWT** (opaque tokens fail). +2. Have `iss` equal to the discovered issuer. +3. Have `aud` equal to **`OIDC_CLIENT_ID`** (string or array containing it). +4. Carry RBAC claims on the **access** token (see [external-oidc-providers.md](external-oidc-providers.md)). +5. Not be a refresh token (`token_use` must not be `refresh`). -**Option A — Client roles (simplest)** -Assign roles on the client matching `OIDC_CLIENT_ID`. The default `roles` scope adds `resource_access` to the token. - -**Option B — `groups` scope** -Create a realm client scope named exactly `groups`. Add a Group Membership or Realm Role mapper that emits a `groups` claim. - -### User identity (User subject) - -Controller reads the User subject from JWT claims (first match): - -`preferred_username` → `username` → `email` → `sub` - -Ensure at least one stable identifier is present for RBAC User bindings. +If `aud` is Microsoft Graph, Okta `api://default`, or any API identifier that is **not** the client ID, Bearer validation fails and APIs return 401. ## Issuer discovery (minimum metadata) @@ -149,43 +169,46 @@ The issuer at `OIDC_ISSUER_URL` must expose: | `userinfo_endpoint` | `GET /user/profile` in external mode | | `revocation_endpoint` | Optional; best-effort logout | -## Keycloak checklist +## Groups, roles, and user identity + +Create IdP groups or application roles named `admin`, `sre`, `developer`, and `viewer`. Those names map **directly** to Controller system roles (no RoleBinding required). Full mapping, claim order, and Keycloak / Entra ID / Okta / Auth0 recipes: [external-oidc-providers.md](external-oidc-providers.md). -Example client: `pot-controller` +**User** subject (first match): `preferred_username` → `username` → `email` → `sub`. -| Setting | Value | -|---------|-------| -| Client authentication | On | -| Standard flow | On | -| Direct access grants | On (if CLI login is required) | -| PKCE Method | S256 | -| Valid redirect URIs | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | -| Web origins | `{CONSOLE_URL}` | -| Client scopes | `openid`, `profile`, `email`, `roles` (default); `groups`, `offline_access` (optional) | +**Group** subjects from the access token (all lowercased): -### MFA and forced password change +1. `resource_access[{OIDC_CLIENT_ID}].roles` +2. Top-level `roles` array +3. Top-level `groups` array + +## MFA and forced password change - **Browser:** enforced by the IdP during authorize (for example Keycloak required action `UPDATE_PASSWORD`) - **CLI:** IdP password-grant policy applies - Controller does not run embedded interaction UI in external mode +## High availability + +For multiple Controller replicas with browser login, set `AUTH_SESSION_STORE_TYPE=database` (mysql/postgres) so OAuth `state` and the PKCE verifier survive any replica. `memory` is only safe for a single replica. + ## Verification ### Browser -1. Viewer Sign in → IdP login page (no `invalid_scope` or PKCE errors) +1. Console Sign in → IdP login page (no `invalid_scope` or PKCE errors) 2. Callback → `{CONSOLE_URL}/login#accessToken=...&refreshToken=...` -3. `GET /api/v3/user/profile` with `Authorization: Bearer ` → 200 +3. Decode the **access** token: `aud` is `OIDC_CLIENT_ID`; `iss` matches the issuer; `roles` / `groups` / `resource_access` contains a system role name +4. `GET /api/v3/user/profile` with `Authorization: Bearer ` → 200 ### CLI ```bash curl -sS -X POST '{CONTROLLER_PUBLIC_URL}/api/v3/user/login' \ -H 'Content-Type: application/json' \ - -d '{"email":"","password":"","totp":""}' + -d '{"email":"","password":"","totp":""}' ``` -Expect `{ "accessToken", "refreshToken" }` (IdP tokens). +`email` is the IdP **login identifier** (username or UPN). Expect `{ "accessToken", "refreshToken" }` (IdP tokens). ### Refresh @@ -199,12 +222,20 @@ curl -sS -X POST '{CONTROLLER_PUBLIC_URL}/api/v3/user/refresh' \ | Error | Likely cause | |-------|----------------| -| `invalid_scope` | Missing `groups` or `offline_access` on the client; wrong scope name (`group` vs `groups`) | +| `invalid_scope` | Missing `groups` or `offline_access` on the client; wrong scope name (`group` vs `groups`); provider requires a full URI scope | | `Missing parameter: code_challenge_method` | PKCE required on IdP but not sent by Controller (upgrade Controller) | -| `redirect_uri` mismatch | Callback URL not registered exactly | +| `redirect_uri` mismatch | Callback URL not registered exactly; URI registered as SPA instead of Web | | Login works, no `refreshToken` in browser hash | `offline_access` not requested or not assigned on the IdP client | -| 403 on API routes | Token valid but RBAC groups/roles not mapped; check client roles or `groups` claim | - -## Related API documentation - -See `docs/swagger.yaml` for `/user/oauth/authorize`, `/user/oauth/callback`, `/user/login`, `/user/refresh`, and `/user/profile`. +| 401 on API routes after login | Opaque access token; `aud` is not `OIDC_CLIENT_ID`; wrong `iss` | +| 403 on API routes | Token valid but RBAC groups/roles not on the **access** token; see [external-oidc-providers.md](external-oidc-providers.md) | +| CLI login fails, browser works | ROPC / direct access grants disabled | +| OAuth `state` errors on HA | `AUTH_SESSION_STORE_TYPE` still `memory` | + +## Related documentation + +| Document | Topic | +|----------|-------| +| [external-oidc-providers.md](external-oidc-providers.md) | Default groups/roles; Keycloak, Entra ID, Okta, Auth0 recipes | +| [oidc-configuration.md](oidc-configuration.md) | Auth modes and environment variables | +| [rbac-reference.md](rbac-reference.md) | System roles, verbs, RoleBindings | +| [swagger.yaml](swagger.yaml) | `/user/oauth/authorize`, `/user/oauth/callback`, `/user/login`, `/user/refresh`, `/user/profile` | diff --git a/docs/external-oidc-providers.md b/docs/external-oidc-providers.md new file mode 100644 index 00000000..8926c6d8 --- /dev/null +++ b/docs/external-oidc-providers.md @@ -0,0 +1,269 @@ +# External OIDC providers — groups, roles, and IdP recipes + +**Audience:** IdP and platform administrators +**Controller mode:** `AUTH_MODE=external` + +This page maps IdP groups and roles onto Controller RBAC and gives setup recipes for common OpenID Connect providers. Register the OIDC **client** first using [external-oidc-client-setup.md](external-oidc-client-setup.md) (confidential client, PKCE S256, redirect URI, scopes, access-token `aud`). + +System role permissions: [rbac-reference.md](rbac-reference.md). + +## Default groups and roles + +Controller ships four **human** system roles. Names are matched **case-insensitively** and compared in **lowercase**. + +| IdP group or role name | Controller system role | Typical use | +|------------------------|------------------------|-------------| +| `admin` | **admin** | Full cluster administration (`resources: ['*']`, `verbs: ['*']`) | +| `sre` | **sre** | Day-2 operations; read-only on `roles`, `roleBindings`, and some NATS operator objects | +| `developer` | **developer** | CRUD on workloads; read-only on infra | +| `viewer` | **viewer** | `get` / `list` only; no exec, tunnels, or events | + +Create these four names as **groups**, **client roles**, or **application roles** on the IdP, then assign users there. That is the intended drop-in setup. + +Two more system roles exist but must **not** be assigned to human IdP users: + +| Name | Purpose | +|------|---------| +| `agent-admin` | Edgelet service accounts (`edgelet.iofog.org/v1`) | +| `microservice` | Limited self-service for running workloads | + +Embedded mode also seeds `AuthGroups` rows named `admin`, `sre`, `developer`, and `viewer`. Those table rows are for **embedded** user membership. In **external** mode they are not consulted for login; JWT claims are the source of Group subjects. + +### Direct mapping (no RoleBinding) + +If the **access** token contains a group or role whose name is exactly `admin`, `sre`, `developer`, or `viewer`, Controller grants that system role immediately. You do not need a RoleBinding. + +Custom IdP names (`Platform Admins`, Entra group object IDs, Keycloak paths like `/admin`) do **not** match. Either: + +- Change the IdP mapper so the claim value is `admin` / `sre` / `developer` / `viewer`, or +- Create a Controller RoleBinding whose Group subject is the **exact lowercase claim value**. + +### How Controller reads the access token + +Group subjects (`src/lib/rbac/middleware.js`), all lowercased: + +1. `resource_access[{OIDC_CLIENT_ID}].roles` — Keycloak-style **client roles** (recommended on Keycloak) +2. Top-level `roles` array — Entra app roles; many Okta/Auth0 mappings +3. Top-level `groups` array — `groups` scope / group mapper + +Namespaced claims such as `https://controller.example.com/groups` are **not** read. Map them to `groups` or `roles`. + +ID-token-only claims are ignored for API authorization. The same claims must appear on the **access** token. + +User subject (first match): `preferred_username` → `username` → `email` → `sub`. + +If you bind a **User** (not Group) RoleBinding, the `name` must match that first non-empty claim. Entra `preferred_username` is usually the UPN. Keycloak’s is often a short username, not email. + +### Optional RoleBindings for custom names + +When the token cannot emit the four system names: + +```yaml +apiVersion: datasance.com/v3 +kind: RoleBinding +metadata: + name: platform-admins-admin +subjects: + - kind: Group + name: platform-admins +roleRef: + kind: Role + name: admin +``` + +User binding (when groups cannot be put in the token): + +```yaml +apiVersion: datasance.com/v3 +kind: RoleBinding +metadata: + name: alice-developer +subjects: + - kind: User + name: alice@example.com +roleRef: + kind: Role + name: developer +``` + +The User `name` must match the JWT user claim described above. + +## Switching from embedded to external + +1. Export users/groups: `POST /api/v3/auth/migration/export` (no password hashes). +2. Create matching users and groups (`admin` / `sre` / `developer` / `viewer`) on the IdP. +3. Switch env to `AUTH_MODE=external` plus issuer and client credentials ([client setup](external-oidc-client-setup.md)). +4. **External → embedded is not supported.** + +Assign at least one operator to `admin` before cutting over. + +--- + +## Provider recipes + +Every recipe assumes the confidential client, redirect URI, and hardcoded scopes from [external-oidc-client-setup.md](external-oidc-client-setup.md): + +```text +openid profile email groups offline_access +``` + +Access token `aud` must equal `OIDC_CLIENT_ID`. + +### Keycloak + +**Issuer** + +```text +https://auth.example.com/realms/{realm} +``` + +**Client** (`pot-controller` or your `OIDC_CLIENT_ID`) + +| Setting | Value | +|---------|-------| +| Client authentication | On (confidential) | +| Standard flow | On | +| Direct access grants | On (if CLI login is required) | +| Implicit | Off | +| PKCE Method | S256 | +| Valid redirect URIs | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | +| Web origins | `{CONSOLE_URL}` | +| Default scopes | `openid`, `profile`, `email`, `roles` | +| Optional scopes | `groups`, `offline_access` | + +**RBAC (pick one)** + +- **Option A (simplest):** Client roles on this client named `admin`, `sre`, `developer`, `viewer`. The default `roles` scope puts them in `resource_access[{clientId}].roles`. +- **Option B:** Realm client scope named exactly `groups`. Group Membership mapper: claim name `groups`, **full group path = OFF** so the claim is `admin`, not `/admin`. + +Realm roles in a top-level `roles` array also work if you add that mapper to the access token. + +### Microsoft Entra ID (Azure AD) + +**Issuer** (v2) + +```text +https://login.microsoftonline.com/{tenant-id}/v2.0 +``` + +**App registration** + +| Setting | Value | +|---------|-------| +| Application type | **Web** (confidential), not SPA | +| Redirect URI | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` (platform **Web**) | +| Certificates & secrets | Client secret → `OIDC_CLIENT_SECRET` | +| `OIDC_CLIENT_ID` | Application (client) ID | +| Allow public client flows | On **only** if you need ROPC for potctl | +| Implicit | Off | + +**Audience.** Entra issues a Microsoft Graph access token if the client only requests OIDC scopes. Controller then rejects `aud`. To make `aud` the application ID: + +1. **Expose an API** on the same app. +2. Prefer an Application ID URI that yields access tokens whose `aud` is the **application (client) ID**. +3. Add a scope whose **short name is `groups`** so the authorize request is accepted. +4. Authorize this same app to that scope. +5. Confirm a decoded access token has `"aud": ""` and is a JWT. + +If Entra will only accept full-URI scopes (`api:///groups`), authorize returns `invalid_scope` because Controller always requests the short name `groups`. + +**RBAC (recommended: App roles, not directory groups).** Directory `groups` claims are often **object GUIDs**, and large tenants hit the groups overage claim (no array). Controller only reads a `groups` **array** of names. + +Define **App roles** (allowed member type: Users/Groups) with **values**: + +```text +admin +sre +developer +viewer +``` + +Assign users or security groups to those app roles. Entra puts them on the access token as: + +```json +"roles": ["admin"] +``` + +Controller maps `roles[]` to Group subjects. No RoleBinding needed. + +**ROPC:** Microsoft is deprecating this grant. Treat CLI password login as optional; prefer Console OAuth. + +### Okta + +**Issuer** (custom authorization server recommended) + +```text +https://{okta-domain}/oauth2/{authorizationServerId} +``` + +**Application** + +| Setting | Value | +|---------|-------| +| Sign-in method | OIDC | +| Application type | **Web** | +| Grant types | Authorization Code, Refresh Token, and Resource Owner Password if CLI is required | +| Client authentication | Client secret | +| Sign-in redirect | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | +| Trusted origins | `{CONSOLE_URL}` as needed | + +**Audience:** the authorization server **Audience** must be the **client ID**. The Okta default `api://default` fails JWT `aud` checks. + +**Scopes:** add a custom scope named `groups` on that authorization server, plus `offline_access`. + +**Claims:** on the **access token**, add claim `groups` (or `roles`) whose value is group names, filtered to `admin`, `sre`, `developer`, `viewer`. ID-token-only claims are not enough. + +Create Okta groups with those same names and assign users. + +### Auth0 (and similar “API + application” IdPs) + +1. Create a **Regular Web Application** (confidential). +2. Create an **API** whose **identifier is exactly `OIDC_CLIENT_ID`** so `aud` matches. +3. On that API, add a permission/scope named `groups`. +4. Enable Authorization Code, Refresh Token, and Password (if CLI). +5. Enable RBAC on the API; add roles named `admin`, `sre`, `developer`, `viewer`. +6. Add a Login / post-login Action that copies roles onto the **access token** as `roles` or `groups` (not a namespaced claim). +7. Without an API audience, Auth0 often returns an **opaque** access token — Controller cannot validate that. + +### Generic OpenID Connect + +Any provider works if it satisfies this contract: + +| Requirement | Value | +|-------------|--------| +| Discovery | `{issuer}/.well-known/openid-configuration` | +| Client | Confidential + secret | +| Code flow | Authorization code + PKCE S256 | +| Refresh | `offline_access` | +| Password grant | Only if CLI login is required | +| Redirect | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | +| Access token | JWT, `aud` = client ID | +| Groups | `resource_access[clientId].roles` **or** `roles[]` **or** `groups[]` with names `admin` / `sre` / `developer` / `viewer` | + +--- + +## Verification (RBAC) + +After [client-setup verification](external-oidc-client-setup.md#verification): + +1. Decode the access token and confirm one of `resource_access`, `roles`, or `groups` contains `admin`, `sre`, `developer`, or `viewer`. +2. Sign in as `viewer`: list applications succeeds; mutating routes return 403. +3. Sign in as `admin`: admin-only routes (for example JWKS rotate is embedded-only; role mutations) succeed per [rbac-reference.md](rbac-reference.md). + +## Troubleshooting (groups and roles) + +| Symptom | Likely cause | +|---------|----------------| +| 403 after a successful login | Claims on the ID token only; Keycloak **full group path** left on (`/admin` ≠ `admin`); Entra group **GUIDs**; Auth0 namespaced claims | +| Login works, no groups in token | `groups` / `roles` mapper not added to the **access** token; client roles not assigned to the user | +| `aud` is Graph or `api://default` | See Entra and Okta audience notes above | +| User RoleBinding never matches | Binding uses email but JWT User subject is `preferred_username` | + +## Related documentation + +| Document | Topic | +|----------|-------| +| [external-oidc-client-setup.md](external-oidc-client-setup.md) | Confidential client, grants, scopes, redirect URI, access-token `aud` | +| [oidc-configuration.md](oidc-configuration.md) | Auth modes and environment variables | +| [rbac-reference.md](rbac-reference.md) | System roles, verbs, custom RoleBindings | +| [swagger.yaml](swagger.yaml) | User and RBAC HTTP APIs | diff --git a/docs/oidc-configuration.md b/docs/oidc-configuration.md index fa91a692..8cb8b3a3 100644 --- a/docs/oidc-configuration.md +++ b/docs/oidc-configuration.md @@ -142,7 +142,7 @@ After login, API calls use `Authorization: Bearer `. Controller re **User** subject: `preferred_username` → `username` → `email` → `sub`. -External IdP setup (scopes, PKCE, redirect URIs, Keycloak checklist): [external-oidc-client-setup.md](external-oidc-client-setup.md). +External IdP client (scopes, PKCE, redirect URIs, access-token `aud`): [external-oidc-client-setup.md](external-oidc-client-setup.md). Default groups/roles and Keycloak / Entra ID / Okta / Auth0 recipes: [external-oidc-providers.md](external-oidc-providers.md). --- @@ -203,7 +203,8 @@ See [external-oidc-client-setup.md](external-oidc-client-setup.md) § Verificati | Document | Topic | |----------|-------| -| [external-oidc-client-setup.md](external-oidc-client-setup.md) | External IdP client, scopes, Keycloak checklist | +| [external-oidc-client-setup.md](external-oidc-client-setup.md) | External IdP client, scopes, redirect URI, access-token `aud` | +| [external-oidc-providers.md](external-oidc-providers.md) | Default groups/roles; Keycloak, Entra ID, Okta, Auth0 recipes | | [rbac-reference.md](rbac-reference.md) | Roles, bindings, public routes | | [architecture.md](architecture.md) | Auth modes summary, API surfaces | | [swagger.yaml](swagger.yaml) | User and auth admin endpoints | diff --git a/docs/rbac-reference.md b/docs/rbac-reference.md index 3e6bec2a..d9d60b21 100644 --- a/docs/rbac-reference.md +++ b/docs/rbac-reference.md @@ -32,7 +32,7 @@ Client request | **User** | `preferred_username`, `username`, `email`, or `sub` | | **Group** | `resource_access[].roles`, `roles[]`, `groups[]` (lowercased) | -Group names should match RoleBinding subjects and IdP role names (for example `admin`, `developer`, `viewer`). See [external-oidc-client-setup.md](external-oidc-client-setup.md) for external IdP claim mapping. +Group names should match RoleBinding subjects and IdP role names (for example `admin`, `developer`, `viewer`). Direct mapping of those names (no RoleBinding) and per-provider recipes: [external-oidc-providers.md](external-oidc-providers.md). Client registration: [external-oidc-client-setup.md](external-oidc-client-setup.md). ### Password-change gate @@ -191,5 +191,6 @@ Optional CI wiring is planned. | Authorizer | `src/lib/rbac/authorizer.js` | | Route inventory (generated) | `node scripts/route-inventory.js` | | Drift script | `scripts/rbac-audit.js`, `npm run rbac-audit` | -| External IdP groups | `docs/external-oidc-client-setup.md` | +| External IdP client | [external-oidc-client-setup.md](external-oidc-client-setup.md) | +| External IdP groups and providers | [external-oidc-providers.md](external-oidc-providers.md) | | HTTP API spec | `docs/swagger.yaml` | From 543e484790bbbae7ef7c46eef2f2b68977ad8ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 00:04:23 +0300 Subject: [PATCH 40/55] Add Knowledge tables, fog links, and catalog rows on the 3.9.0 schema. --- src/data/managers/fleet-knowledge-manager.js | 33 ++++++++ src/data/managers/fog-knowledge-manager.js | 12 +++ .../microservice-knowledge-item-manager.js | 12 +++ .../microservice-knowledge-manager.js | 12 +++ src/data/managers/microservice-manager.js | 14 ++++ .../db_migration_mysql_baseline_v3.9.0.sql | 53 +++++++++++++ .../mysql/db_migration_mysql_v3.9.0.sql | 55 +++++++++++++ .../db_migration_pg_baseline_v3.9.0.sql | 52 +++++++++++++ .../postgres/db_migration_pg_v3.9.0.sql | 54 +++++++++++++ .../db_migration_sqlite_baseline_v3.9.0.sql | 52 +++++++++++++ .../sqlite/db_migration_sqlite_v3.9.0.sql | 54 +++++++++++++ src/data/models/changetracking.js | 10 +++ src/data/models/fleet-knowledge.js | 78 +++++++++++++++++++ src/data/models/fog-knowledge.js | 10 +++ src/data/models/fog.js | 25 ++++++ .../models/microservice-knowledge-item.js | 41 ++++++++++ src/data/models/microservice-knowledge.js | 35 +++++++++ src/data/models/microservice.js | 10 +++ 18 files changed, 612 insertions(+) create mode 100644 src/data/managers/fleet-knowledge-manager.js create mode 100644 src/data/managers/fog-knowledge-manager.js create mode 100644 src/data/managers/microservice-knowledge-item-manager.js create mode 100644 src/data/managers/microservice-knowledge-manager.js create mode 100644 src/data/models/fleet-knowledge.js create mode 100644 src/data/models/fog-knowledge.js create mode 100644 src/data/models/microservice-knowledge-item.js create mode 100644 src/data/models/microservice-knowledge.js diff --git a/src/data/managers/fleet-knowledge-manager.js b/src/data/managers/fleet-knowledge-manager.js new file mode 100644 index 00000000..ae6812e4 --- /dev/null +++ b/src/data/managers/fleet-knowledge-manager.js @@ -0,0 +1,33 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const FleetKnowledge = models.FleetKnowledge + +const knowledgeExcludedFields = [ + 'created_at', + 'updated_at' +] + +class FleetKnowledgeManager extends BaseManager { + getEntity () { + return FleetKnowledge + } + + findOne (where, transaction) { + return FleetKnowledge.findOne({ + where, + attributes: { exclude: knowledgeExcludedFields }, + transaction + }) + } + + findAll (where, transaction) { + return FleetKnowledge.findAll({ + where, + attributes: { exclude: knowledgeExcludedFields }, + transaction + }) + } +} + +const instance = new FleetKnowledgeManager() +module.exports = instance diff --git a/src/data/managers/fog-knowledge-manager.js b/src/data/managers/fog-knowledge-manager.js new file mode 100644 index 00000000..fbe57ecb --- /dev/null +++ b/src/data/managers/fog-knowledge-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const FogKnowledge = models.FogKnowledge + +class FogKnowledgeManager extends BaseManager { + getEntity () { + return FogKnowledge + } +} + +const instance = new FogKnowledgeManager() +module.exports = instance diff --git a/src/data/managers/microservice-knowledge-item-manager.js b/src/data/managers/microservice-knowledge-item-manager.js new file mode 100644 index 00000000..7830672b --- /dev/null +++ b/src/data/managers/microservice-knowledge-item-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceKnowledgeItem = models.MicroserviceKnowledgeItem + +class MicroserviceKnowledgeItemManager extends BaseManager { + getEntity () { + return MicroserviceKnowledgeItem + } +} + +const instance = new MicroserviceKnowledgeItemManager() +module.exports = instance diff --git a/src/data/managers/microservice-knowledge-manager.js b/src/data/managers/microservice-knowledge-manager.js new file mode 100644 index 00000000..505c5348 --- /dev/null +++ b/src/data/managers/microservice-knowledge-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceKnowledge = models.MicroserviceKnowledge + +class MicroserviceKnowledgeManager extends BaseManager { + getEntity () { + return MicroserviceKnowledge + } +} + +const instance = new MicroserviceKnowledgeManager() +module.exports = instance diff --git a/src/data/managers/microservice-manager.js b/src/data/managers/microservice-manager.js index 72576c70..f918c3db 100644 --- a/src/data/managers/microservice-manager.js +++ b/src/data/managers/microservice-manager.js @@ -15,6 +15,8 @@ const MicroserviceTmpfs = models.MicroserviceTmpfs const MicroserviceUlimit = models.MicroserviceUlimit const MicroserviceModel = models.MicroserviceModel const MicroserviceModelItem = models.MicroserviceModelItem +const MicroserviceKnowledge = models.MicroserviceKnowledge +const MicroserviceKnowledgeItem = models.MicroserviceKnowledgeItem const CatalogItem = models.CatalogItem const CatalogItemImage = models.CatalogItemImage const Fog = models.Fog @@ -74,6 +76,18 @@ function containerChildIncludes () { as: 'modelItems', required: false, attributes: ['id', 'name'] + }, + { + model: MicroserviceKnowledge, + as: 'microserviceKnowledge', + required: false, + attributes: ['bindPath', 'permissions'] + }, + { + model: MicroserviceKnowledgeItem, + as: 'knowledgeItems', + required: false, + attributes: ['id', 'name'] } ] } diff --git a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql index 45d3c144..780894bf 100644 --- a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql @@ -126,6 +126,9 @@ CREATE TABLE IF NOT EXISTS Fogs ( available_cdi_devices TEXT, active_models BIGINT DEFAULT 0, model_last_update BIGINT DEFAULT 0, + knowledge_status TEXT, + active_knowledge BIGINT DEFAULT 0, + knowledge_last_update BIGINT DEFAULT 0, FOREIGN KEY (arch_id) REFERENCES Architectures (id) ); @@ -152,6 +155,8 @@ CREATE TABLE IF NOT EXISTS ChangeTrackings ( models BOOLEAN DEFAULT false, runtime_classes BOOLEAN DEFAULT false, microservice_models BOOLEAN DEFAULT false, + knowledge BOOLEAN DEFAULT false, + microservice_knowledge BOOLEAN DEFAULT false, FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE ); @@ -1359,4 +1364,52 @@ CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( CREATE INDEX idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); CREATE INDEX idx_microservice_model_items_name ON MicroserviceModelItems (name); +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); + COMMIT; diff --git a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql index e116a7d9..c307d22c 100644 --- a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql @@ -178,3 +178,58 @@ ALTER TABLE VolumeMappings ADD COLUMN scope VARCHAR(16) NOT NULL DEFAULT 'privat DROP TABLE IF EXISTS HWInfos; DROP TABLE IF EXISTS USBInfos; + +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); + +ALTER TABLE ChangeTrackings ADD COLUMN knowledge BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_knowledge BOOLEAN DEFAULT false; + +ALTER TABLE Fogs ADD COLUMN knowledge_status TEXT; +ALTER TABLE Fogs ADD COLUMN active_knowledge BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN knowledge_last_update BIGINT DEFAULT 0; diff --git a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql index fff2c7d2..64bb3ec2 100644 --- a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql @@ -124,6 +124,9 @@ CREATE TABLE IF NOT EXISTS "Fogs" ( available_cdi_devices TEXT, active_models BIGINT DEFAULT 0, model_last_update BIGINT DEFAULT 0, + knowledge_status TEXT, + active_knowledge BIGINT DEFAULT 0, + knowledge_last_update BIGINT DEFAULT 0, FOREIGN KEY (arch_id) REFERENCES "Architectures" (id) ); @@ -150,6 +153,8 @@ CREATE TABLE IF NOT EXISTS "ChangeTrackings" ( models BOOLEAN DEFAULT false, runtime_classes BOOLEAN DEFAULT false, microservice_models BOOLEAN DEFAULT false, + knowledge BOOLEAN DEFAULT false, + microservice_knowledge BOOLEAN DEFAULT false, FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE ); @@ -1347,3 +1352,50 @@ CREATE TABLE IF NOT EXISTS "MicroserviceModelItems" ( CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON "MicroserviceModelItems" (microservice_uuid); CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON "MicroserviceModelItems" (name); + +CREATE TABLE IF NOT EXISTS "Knowledge" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON "Knowledge" (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON "Knowledge" (name); + +CREATE TABLE IF NOT EXISTS "FogKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES "Knowledge" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON "FogKnowledge" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON "FogKnowledge" (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledgeItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Knowledge" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON "MicroserviceKnowledgeItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON "MicroserviceKnowledgeItems" (name); diff --git a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql index 48a6c3a7..d1f2ffe9 100644 --- a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql @@ -175,3 +175,57 @@ ALTER TABLE "VolumeMappings" ADD COLUMN IF NOT EXISTS scope VARCHAR(16) NOT NULL DROP TABLE IF EXISTS "HWInfos"; DROP TABLE IF EXISTS "USBInfos"; + +CREATE TABLE IF NOT EXISTS "Knowledge" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON "Knowledge" (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON "Knowledge" (name); + +CREATE TABLE IF NOT EXISTS "FogKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES "Knowledge" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON "FogKnowledge" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON "FogKnowledge" (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledgeItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Knowledge" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON "MicroserviceKnowledgeItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON "MicroserviceKnowledgeItems" (name); + +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS knowledge BOOLEAN DEFAULT false; +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS microservice_knowledge BOOLEAN DEFAULT false; + +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS knowledge_status TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS active_knowledge BIGINT DEFAULT 0; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS knowledge_last_update BIGINT DEFAULT 0; diff --git a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql index 55a2c030..cc38a04f 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql @@ -124,6 +124,9 @@ CREATE TABLE IF NOT EXISTS Fogs ( available_cdi_devices TEXT, active_models BIGINT DEFAULT 0, model_last_update BIGINT DEFAULT 0, + knowledge_status TEXT, + active_knowledge BIGINT DEFAULT 0, + knowledge_last_update BIGINT DEFAULT 0, FOREIGN KEY (arch_id) REFERENCES Architectures (id) ); @@ -150,6 +153,8 @@ CREATE TABLE IF NOT EXISTS ChangeTrackings ( models BOOLEAN DEFAULT false, runtime_classes BOOLEAN DEFAULT false, microservice_models BOOLEAN DEFAULT false, + knowledge BOOLEAN DEFAULT false, + microservice_knowledge BOOLEAN DEFAULT false, FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE ); @@ -1346,3 +1351,50 @@ CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON MicroserviceModelItems (name); + +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); diff --git a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql index 0e8c8922..420751ba 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql @@ -192,3 +192,57 @@ ALTER TABLE VolumeMappings ADD COLUMN scope TEXT NOT NULL DEFAULT 'private'; DROP TABLE IF EXISTS HWInfos; DROP TABLE IF EXISTS USBInfos; + +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); + +ALTER TABLE ChangeTrackings ADD COLUMN knowledge BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_knowledge BOOLEAN DEFAULT false; + +ALTER TABLE Fogs ADD COLUMN knowledge_status TEXT; +ALTER TABLE Fogs ADD COLUMN active_knowledge BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN knowledge_last_update BIGINT DEFAULT 0; diff --git a/src/data/models/changetracking.js b/src/data/models/changetracking.js index e5bae7be..e1170f66 100644 --- a/src/data/models/changetracking.js +++ b/src/data/models/changetracking.js @@ -93,6 +93,16 @@ module.exports = (sequelize, DataTypes) => { field: 'microservice_models', defaultValue: false }, + knowledge: { + type: DataTypes.BOOLEAN, + field: 'knowledge', + defaultValue: false + }, + microserviceKnowledge: { + type: DataTypes.BOOLEAN, + field: 'microservice_knowledge', + defaultValue: false + }, lastUpdated: { type: DataTypes.STRING, field: 'last_updated', diff --git a/src/data/models/fleet-knowledge.js b/src/data/models/fleet-knowledge.js new file mode 100644 index 00000000..2e91ece2 --- /dev/null +++ b/src/data/models/fleet-knowledge.js @@ -0,0 +1,78 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FleetKnowledge = sequelize.define('FleetKnowledge', { + uuid: { + type: DataTypes.STRING(36), + primaryKey: true, + allowNull: false, + field: 'uuid' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + field: 'name' + }, + repo: { + type: DataTypes.TEXT, + field: 'repo' + }, + revision: { + type: DataTypes.TEXT, + field: 'revision', + defaultValue: '' + }, + files: { + type: DataTypes.TEXT, + field: 'files', + defaultValue: '[]', + get () { + const value = this.getDataValue('files') + if (value == null || value === '') { + return [] + } + try { + return JSON.parse(value) + } catch (e) { + return [] + } + }, + set (value) { + if (value == null) { + this.setDataValue('files', '[]') + return + } + this.setDataValue('files', typeof value === 'string' ? value : JSON.stringify(value)) + } + }, + format: { + type: DataTypes.STRING, + field: 'format' + } + }, { + tableName: 'Knowledge', + timestamps: true, + underscored: true + }) + + FleetKnowledge.associate = function (models) { + FleetKnowledge.belongsTo(models.Registry, { + foreignKey: { + name: 'registryId', + field: 'registry_id' + }, + as: 'registry', + onDelete: 'set null' + }) + + FleetKnowledge.belongsToMany(models.Fog, { + through: models.FogKnowledge, + as: 'fogs', + foreignKey: 'knowledge_uuid', + otherKey: 'fog_uuid' + }) + } + + return FleetKnowledge +} diff --git a/src/data/models/fog-knowledge.js b/src/data/models/fog-knowledge.js new file mode 100644 index 00000000..dbf83460 --- /dev/null +++ b/src/data/models/fog-knowledge.js @@ -0,0 +1,10 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FogKnowledge = sequelize.define('FogKnowledge', {}, { + tableName: 'FogKnowledge', + timestamps: false, + underscored: true + }) + return FogKnowledge +} diff --git a/src/data/models/fog.js b/src/data/models/fog.js index e3d73885..ca8ddadd 100644 --- a/src/data/models/fog.js +++ b/src/data/models/fog.js @@ -355,6 +355,25 @@ module.exports = (sequelize, DataTypes) => { }, field: 'model_last_update' }, + knowledgeStatus: { + type: DataTypes.TEXT, + field: 'knowledge_status' + }, + activeKnowledge: { + type: DataTypes.BIGINT, + defaultValue: 0, + get () { + return convertToInt(this.getDataValue('activeKnowledge'), 0) + }, + field: 'active_knowledge' + }, + knowledgeLastUpdate: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('knowledgeLastUpdate'), 0) + }, + field: 'knowledge_last_update' + }, warningMessage: { type: DataTypes.TEXT, field: 'warning_message', @@ -427,6 +446,12 @@ module.exports = (sequelize, DataTypes) => { foreignKey: 'fog_uuid', otherKey: 'model_uuid' }) + Fog.belongsToMany(models.FleetKnowledge, { + through: models.FogKnowledge, + as: 'knowledge', + foreignKey: 'fog_uuid', + otherKey: 'knowledge_uuid' + }) Fog.belongsToMany(models.RuntimeClass, { through: models.FogRuntimeClasses, as: 'runtimeClassLinks', diff --git a/src/data/models/microservice-knowledge-item.js b/src/data/models/microservice-knowledge-item.js new file mode 100644 index 00000000..53c29de7 --- /dev/null +++ b/src/data/models/microservice-knowledge-item.js @@ -0,0 +1,41 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceKnowledgeItem = sequelize.define('MicroserviceKnowledgeItem', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + field: 'name' + } + }, { + tableName: 'MicroserviceKnowledgeItems', + timestamps: false, + underscored: true + }) + MicroserviceKnowledgeItem.associate = function (models) { + MicroserviceKnowledgeItem.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + MicroserviceKnowledgeItem.belongsTo(models.FleetKnowledge, { + foreignKey: { + name: 'name', + field: 'name' + }, + targetKey: 'name', + as: 'knowledge', + onDelete: 'restrict' + }) + } + return MicroserviceKnowledgeItem +} diff --git a/src/data/models/microservice-knowledge.js b/src/data/models/microservice-knowledge.js new file mode 100644 index 00000000..22d438cb --- /dev/null +++ b/src/data/models/microservice-knowledge.js @@ -0,0 +1,35 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceKnowledge = sequelize.define('MicroserviceKnowledge', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + bindPath: { + type: DataTypes.TEXT, + field: 'bind_path' + }, + permissions: { + type: DataTypes.TEXT, + field: 'permissions' + } + }, { + tableName: 'MicroserviceKnowledge', + timestamps: false, + underscored: true + }) + MicroserviceKnowledge.associate = function (models) { + MicroserviceKnowledge.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceKnowledge +} diff --git a/src/data/models/microservice.js b/src/data/models/microservice.js index 2529eb91..4ed5a1b3 100644 --- a/src/data/models/microservice.js +++ b/src/data/models/microservice.js @@ -292,6 +292,16 @@ module.exports = (sequelize, DataTypes) => { as: 'modelItems' }) + Microservice.hasOne(models.MicroserviceKnowledge, { + foreignKey: 'microservice_uuid', + as: 'microserviceKnowledge' + }) + + Microservice.hasMany(models.MicroserviceKnowledgeItem, { + foreignKey: 'microservice_uuid', + as: 'knowledgeItems' + }) + Microservice.hasMany(models.MicroserviceCdiDev, { foreignKey: 'microservice_uuid', as: 'cdiDevices' From 757dd26247e5d109742a7e9bbfc1c67590760fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 00:04:27 +0300 Subject: [PATCH 41/55] Return linked Knowledge to the agent and store knowledge status on fog updates. --- src/controllers/agent-controller.js | 5 ++ src/routes/agent.js | 29 +++++++++++ src/schemas/agent.js | 3 ++ src/services/agent-service.js | 64 +++++++++++++++++++++++-- src/services/change-tracking-service.js | 10 +++- src/services/iofog-service.js | 18 +++++++ 6 files changed, 125 insertions(+), 4 deletions(-) diff --git a/src/controllers/agent-controller.js b/src/controllers/agent-controller.js index f353ac19..f798cd18 100644 --- a/src/controllers/agent-controller.js +++ b/src/controllers/agent-controller.js @@ -56,6 +56,10 @@ const getAgentLinkedModelsEndpoint = async function (req, fog) { return { models: await AgentService.getAgentLinkedModels(fog) } } +const getAgentLinkedKnowledgeEndpoint = async function (req, fog) { + return { knowledge: await AgentService.getAgentLinkedKnowledge(fog) } +} + const getAgentLinkedRuntimeClassesEndpoint = async function (req, fog) { return { runtimeClasses: await AgentService.getAgentLinkedRuntimeClasses(fog) } } @@ -115,6 +119,7 @@ module.exports = { resetAgentConfigChangesEndPoint: AuthDecorator.checkFogToken(resetAgentConfigChangesEndPoint), getAgentLinkedVolumeMountsEndpoint: AuthDecorator.checkFogToken(getAgentLinkedVolumeMountsEndpoint), getAgentLinkedModelsEndpoint: AuthDecorator.checkFogToken(getAgentLinkedModelsEndpoint), + getAgentLinkedKnowledgeEndpoint: AuthDecorator.checkFogToken(getAgentLinkedKnowledgeEndpoint), getAgentLinkedRuntimeClassesEndpoint: AuthDecorator.checkFogToken(getAgentLinkedRuntimeClassesEndpoint), getControllerCAEndPoint: AuthDecorator.checkFogToken(getControllerCAEndPoint), getAgentLogSessionsEndPoint: AuthDecorator.checkFogToken(getAgentLogSessionsEndPoint), diff --git a/src/routes/agent.js b/src/routes/agent.js index a64b680a..2d438d18 100644 --- a/src/routes/agent.js +++ b/src/routes/agent.js @@ -327,6 +327,35 @@ module.exports = [ logger.apiRes({ req, res, responseObject }) } }, + { + method: 'get', + path: '/api/v3/agent/knowledge', + middleware: async (req, res) => { + logger.apiReq(req) + + const successCode = constants.HTTP_CODE_SUCCESS + const errorCodes = [ + { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AgentAuthenticationError, Errors.AuthenticationError] + }, + { + code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, + errors: [Errors.ServiceUnavailableError] + } + ] + + const getAgentLinkedKnowledgeEndpoint = ResponseDecorator.handleErrors(AgentController.getAgentLinkedKnowledgeEndpoint, + successCode, errorCodes) + const responseObject = await getAgentLinkedKnowledgeEndpoint(req) + + res + .status(responseObject.code) + .send(responseObject.body) + + logger.apiRes({ req, res, responseObject }) + } + }, { method: 'get', path: '/api/v3/agent/runtimeClasses', diff --git a/src/schemas/agent.js b/src/schemas/agent.js index 10848df3..af4bcdbe 100644 --- a/src/schemas/agent.js +++ b/src/schemas/agent.js @@ -104,6 +104,9 @@ const updateAgentStatus = { modelStatus: { type: 'string' }, activeModels: { type: 'integer', minimum: 0 }, modelLastUpdate: { type: 'integer', minimum: 0 }, + knowledgeStatus: { type: ['string', 'array'] }, + activeKnowledge: { type: 'integer', minimum: 0 }, + knowledgeLastUpdate: { type: 'integer', minimum: 0 }, runtimeAgentPhase: { type: 'string' }, controlPlaneQuiesced: { type: 'boolean' }, gpsMode: { type: 'string' }, diff --git a/src/services/agent-service.js b/src/services/agent-service.js index b81fdbdd..5af9f135 100644 --- a/src/services/agent-service.js +++ b/src/services/agent-service.js @@ -61,7 +61,9 @@ const CHANGE_TRACKING_KEYS = [ 'fogLogs', 'models', 'runtimeClasses', - 'microserviceModels' + 'microserviceModels', + 'knowledge', + 'microserviceKnowledge' ] for (const key of CHANGE_TRACKING_KEYS) { CHANGE_TRACKING_DEFAULT[key] = false @@ -302,7 +304,10 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { availableCdiDevices: _stringifyStatusField(agentStatus.availableCdiDevices), modelStatus: _stringifyStatusField(agentStatus.modelStatus), activeModels: agentStatus.activeModels, - modelLastUpdate: agentStatus.modelLastUpdate + modelLastUpdate: agentStatus.modelLastUpdate, + knowledgeStatus: _stringifyStatusField(agentStatus.knowledgeStatus), + activeKnowledge: agentStatus.activeKnowledge, + knowledgeLastUpdate: agentStatus.knowledgeLastUpdate } fogStatus = AppHelper.deleteUndefinedFields(fogStatus) @@ -575,7 +580,8 @@ const getAgentMicroservices = async function (fog, transaction) { devices: CatalogContainer.devicesFromRows(microservice.devices), tmpfs: CatalogContainer.tmpfsFromRows(microservice.tmpfs), ulimits: CatalogContainer.ulimitsFromRows(microservice.ulimits), - models: CatalogContainer.catalogFromRows(microservice.microserviceModel, microservice.modelItems) + models: CatalogContainer.catalogFromRows(microservice.microserviceModel, microservice.modelItems), + knowledge: CatalogContainer.catalogFromRows(microservice.microserviceKnowledge, microservice.knowledgeItems) }) // Resolve service account with rules from relationship @@ -900,6 +906,57 @@ const getAgentLinkedModels = async function (fog, transaction) { return models } +function _knowledgeFiles (value) { + if (Array.isArray(value)) { + return value + } + if (value == null || value === '') { + return [] + } + if (typeof value === 'string') { + try { + const parsed = JSON.parse(value) + return Array.isArray(parsed) ? parsed : [] + } catch (e) { + return [] + } + } + return [] +} + +const getAgentLinkedKnowledge = async function (fog, transaction) { + if (!fog || typeof fog.getKnowledge !== 'function') { + return [] + } + + const resources = await fog.getKnowledge({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction + }) + + const knowledge = [] + for (const resource of resources) { + const row = _toPlain(resource) + if (!row || !row.uuid || !row.name) { + continue + } + const item = { + uuid: row.uuid, + name: row.name, + repo: row.repo, + revision: row.revision != null ? row.revision : '', + registryId: row.registryId, + files: _knowledgeFiles(row.files) + } + if (row.format != null && row.format !== '') { + item.format = row.format + } + knowledge.push(item) + } + return knowledge +} + const getAgentLinkedRuntimeClasses = async function (fog, transaction) { if (!fog || typeof fog.getRuntimeClassLinks !== 'function') { return [] @@ -942,6 +999,7 @@ module.exports = { deleteNode: TransactionDecorator.generateTransaction(deleteNode), getAgentLinkedVolumeMounts: TransactionDecorator.generateTransaction(getAgentLinkedVolumeMounts), getAgentLinkedModels: TransactionDecorator.generateTransaction(getAgentLinkedModels), + getAgentLinkedKnowledge: TransactionDecorator.generateTransaction(getAgentLinkedKnowledge), getAgentLinkedRuntimeClasses: TransactionDecorator.generateTransaction(getAgentLinkedRuntimeClasses), getControllerCA, getAgentLogSessions: TransactionDecorator.generateTransaction(getAgentLogSessions), diff --git a/src/services/change-tracking-service.js b/src/services/change-tracking-service.js index 7ab2eff0..7deb2421 100644 --- a/src/services/change-tracking-service.js +++ b/src/services/change-tracking-service.js @@ -19,7 +19,9 @@ const events = Object.freeze({ execSessions: false, models: false, runtimeClasses: false, - microserviceModels: false + microserviceModels: false, + knowledge: false, + microserviceKnowledge: false }, microserviceFull: { microserviceConfig: true, @@ -81,6 +83,12 @@ const events = Object.freeze({ }, microserviceModels: { microserviceModels: true + }, + knowledge: { + knowledge: true + }, + microserviceKnowledge: { + microserviceKnowledge: true } }) diff --git a/src/services/iofog-service.js b/src/services/iofog-service.js index c602d6a1..037df383 100644 --- a/src/services/iofog-service.js +++ b/src/services/iofog-service.js @@ -825,6 +825,21 @@ async function _getFogVolumeMounts (fog, transaction) { }) } +function _parseKnowledgeStatus (value) { + if (typeof value !== 'string') { + return Array.isArray(value) ? value : value + } + if (value === '') { + return [] + } + try { + const parsed = JSON.parse(value) + return Array.isArray(parsed) ? parsed : [] + } catch (e) { + return [] + } +} + async function _getFogExtraInformation (fog, transaction, options = {}) { const fogUuid = fog.uuid const parsedSpec = await FogPlatformSpecManager.getParsedSpec(fogUuid, transaction) @@ -846,6 +861,9 @@ async function _getFogExtraInformation (fog, transaction, options = {}) { } : undefined const result = { ...fogFields, archId, arch, tags: _mapTags(fog), ...routerConfig, ...natsConfig, volumeMounts } + if (Object.prototype.hasOwnProperty.call(result, 'knowledgeStatus')) { + result.knowledgeStatus = _parseKnowledgeStatus(result.knowledgeStatus) + } if (options.includePlatformStatus) { const status = await FogPlatformStatusManager.getParsedStatus(fogUuid, transaction) result.platformStatus = _formatPlatformStatus(status, parsedSpec ? parsedSpec.generation : null) From 41cc95ab8b06416dfb4411f48abb56ff2a611d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 00:04:32 +0300 Subject: [PATCH 42/55] Add fleet Knowledge create, update, YAML, and per-fog link. --- src/controllers/knowledge-controller.js | 63 +++++ src/helpers/error-messages.js | 17 +- src/helpers/knowledge-format.js | 36 +++ src/routes/knowledge.js | 187 +++++++++++++++ src/schemas/knowledge.js | 97 ++++++++ src/services/event-service.js | 1 + src/services/knowledge-service.js | 293 ++++++++++++++++++++++++ src/services/yaml-parser-service.js | 54 +++++ 8 files changed, 747 insertions(+), 1 deletion(-) create mode 100644 src/controllers/knowledge-controller.js create mode 100644 src/helpers/knowledge-format.js create mode 100644 src/routes/knowledge.js create mode 100644 src/schemas/knowledge.js create mode 100644 src/services/knowledge-service.js diff --git a/src/controllers/knowledge-controller.js b/src/controllers/knowledge-controller.js new file mode 100644 index 00000000..a6743c61 --- /dev/null +++ b/src/controllers/knowledge-controller.js @@ -0,0 +1,63 @@ +const KnowledgeService = require('../services/knowledge-service') +const YAMLParserService = require('../services/yaml-parser-service') + +const listKnowledgeEndpoint = async (req) => { + return KnowledgeService.listKnowledgeEndpoint() +} + +const getKnowledgeEndpoint = async (req) => { + return KnowledgeService.getKnowledgeEndpoint(req.params.name) +} + +const createKnowledgeEndpoint = async (req) => { + return KnowledgeService.createKnowledgeEndpoint(req.body) +} + +const updateKnowledgeEndpoint = async (req) => { + return KnowledgeService.updateKnowledgeEndpoint(req.params.name, req.body) +} + +const deleteKnowledgeEndpoint = async (req) => { + return KnowledgeService.deleteKnowledgeEndpoint(req.params.name) +} + +const createKnowledgeYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const knowledgeData = await YAMLParserService.parseKnowledgeFile(fileContent) + return KnowledgeService.createKnowledgeEndpoint(knowledgeData) +} + +const upsertKnowledgeYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const name = req.params.name + const knowledgeData = await YAMLParserService.parseKnowledgeFile(fileContent, { + isUpdate: true, + knowledgeName: name + }) + return KnowledgeService.upsertKnowledgeEndpoint(name, knowledgeData) +} + +const getKnowledgeLinkEndpoint = async (req) => { + return KnowledgeService.getKnowledgeLinkEndpoint(req.params.name) +} + +const linkKnowledgeEndpoint = async (req) => { + return KnowledgeService.linkKnowledgeEndpoint(req.params.name, req.body.fogUuids) +} + +const unlinkKnowledgeEndpoint = async (req) => { + return KnowledgeService.unlinkKnowledgeEndpoint(req.params.name, req.body.fogUuids) +} + +module.exports = { + listKnowledgeEndpoint, + getKnowledgeEndpoint, + createKnowledgeEndpoint, + updateKnowledgeEndpoint, + deleteKnowledgeEndpoint, + createKnowledgeYamlEndpoint, + upsertKnowledgeYamlEndpoint, + getKnowledgeLinkEndpoint, + linkKnowledgeEndpoint, + unlinkKnowledgeEndpoint +} diff --git a/src/helpers/error-messages.js b/src/helpers/error-messages.js index 768bb7cb..6552df0c 100644 --- a/src/helpers/error-messages.js +++ b/src/helpers/error-messages.js @@ -143,6 +143,11 @@ module.exports = { MODEL_UNLINK_IN_USE: 'Cannot unlink model \'{}\' from agent {}: bound by microservices {}', MODEL_HF_FILES_REQUIRED: 'Hugging Face models require a non-empty files list', MODEL_OCI_FILES_FORBIDDEN: 'OCI models must not specify files', + KNOWLEDGE_NOT_FOUND: 'Knowledge with name {} not found', + KNOWLEDGE_ALREADY_EXISTS: 'Knowledge with name {} already exists', + KNOWLEDGE_IN_USE: 'Cannot delete knowledge \'{}\': bound by microservices {}', + KNOWLEDGE_UNLINK_IN_USE: 'Cannot unlink knowledge \'{}\' from agent {}: bound by microservices {}', + KNOWLEDGE_OCI_FILES_FORBIDDEN: 'OCI knowledge must not specify files', RUNTIME_CLASS_NOT_FOUND: 'RuntimeClass with name {} not found', RUNTIME_CLASS_ALREADY_EXISTS: 'RuntimeClass with name {} already exists', RUNTIME_CLASS_IN_USE: 'Cannot delete RuntimeClass \'{}\': pinned by microservices {}', @@ -151,12 +156,22 @@ module.exports = { RUNTIME_CLASS_LINK_NOT_AVAILABLE: 'Cannot link RuntimeClass \'{}\' to agent \'{}\': runtime is not in availableRuntimes', RUNTIME_CLASS_MISSING: 'RuntimeClass \'{}\' does not exist', MICROSERVICE_CATALOG_BIND_PATH_REQUIRED: 'Model catalog bindPath is required when items are present', + MICROSERVICE_CATALOG_BIND_PATH_REQUIRES_ITEMS: 'Model catalog bindPath requires at least one item', MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE: 'Model catalog bindPath must be an absolute container path', MICROSERVICE_CATALOG_PERMISSIONS: 'Model catalog permissions must be ro or rw', MICROSERVICE_CATALOG_DUPLICATE_ITEM: 'Model catalog item name \'{}\' is duplicated', MICROSERVICE_CATALOG_ITEM_NAME: 'Model catalog item name \'{}\' is invalid', - MICROSERVICE_CATALOG_PATH_COLLISION: 'Model catalog path \'{}\' collides with a volume or tmpfs mount', + MICROSERVICE_CATALOG_PATH_COLLISION: 'Model catalog path \'{}\' collides with a volume, tmpfs, or knowledge catalog mount', MICROSERVICE_CATALOG_MODEL_NOT_FOUND: 'Model catalog item \'{}\' does not match a fleet model', + MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRED: 'Knowledge catalog bindPath is required when items are present', + MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRES_ITEMS: 'Knowledge catalog bindPath requires at least one item', + MICROSERVICE_KNOWLEDGE_BIND_PATH_ABSOLUTE: 'Knowledge catalog bindPath must be an absolute container path', + MICROSERVICE_KNOWLEDGE_PERMISSIONS: 'Knowledge catalog permissions must be ro or rw', + MICROSERVICE_KNOWLEDGE_DUPLICATE_ITEM: 'Knowledge catalog item name \'{}\' is duplicated', + MICROSERVICE_KNOWLEDGE_ITEM_NAME: 'Knowledge catalog item name \'{}\' is invalid', + MICROSERVICE_KNOWLEDGE_PATH_COLLISION: 'Knowledge catalog path \'{}\' collides with a volume, tmpfs, or model catalog mount', + MICROSERVICE_KNOWLEDGE_NOT_FOUND: 'Knowledge catalog item \'{}\' does not match fleet knowledge', + MICROSERVICE_KNOWLEDGE_SYSTEM_FORBIDDEN: 'System microservices cannot set a knowledge catalog', MICROSERVICE_RUN_AS_USER_GROUP_CONFLICT: 'runAsUser cannot contain \':\' when runAsGroup is set', MICROSERVICE_MEMORY_SWAP_REQUIRES_LIMIT: 'memorySwap requires memoryLimit unless it is -1', MICROSERVICE_MEMORY_SWAP_VALUE: 'memorySwap must be -1 or a positive MiB total', diff --git a/src/helpers/knowledge-format.js b/src/helpers/knowledge-format.js new file mode 100644 index 00000000..34f255c6 --- /dev/null +++ b/src/helpers/knowledge-format.js @@ -0,0 +1,36 @@ +const KNOWN_FORMATS = new Set([ + 'markdown', + 'pdf', + 'jsonl', + 'parquet', + 'arrow', + 'sqlite', + 'faiss', + 'chroma', + 'lance' +]) + +/** + * Soft hint for a Knowledge artifact. Never throws. + * Omit, null, and empty string stay unset. Known tokens persist lowercase. + * Any other value persists "unknown". + * @param {*} value + * @returns {string|null} + */ +function normalizeKnowledgeFormat (value) { + if (value == null || value === '') { + return null + } + if (typeof value !== 'string') { + return 'unknown' + } + const token = value.toLowerCase() + if (KNOWN_FORMATS.has(token)) { + return token + } + return 'unknown' +} + +module.exports = { + normalizeKnowledgeFormat +} diff --git a/src/routes/knowledge.js b/src/routes/knowledge.js new file mode 100644 index 00000000..6c0c31ab --- /dev/null +++ b/src/routes/knowledge.js @@ -0,0 +1,187 @@ +const constants = require('../helpers/constants') +const KnowledgeController = require('../controllers/knowledge-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/knowledge', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.listKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/knowledge/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.getKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/knowledge/:name', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.updateKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/knowledge/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.deleteKnowledgeEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/knowledge', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.createKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/knowledge/yaml', + fileInput: 'knowledge', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.createKnowledgeYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/knowledge/yaml/:name', + fileInput: 'knowledge', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.upsertKnowledgeYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'get', + path: '/api/v3/knowledge/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.getKnowledgeLinkEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/knowledge/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.linkKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/knowledge/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.unlinkKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/schemas/knowledge.js b/src/schemas/knowledge.js new file mode 100644 index 00000000..6930dc7a --- /dev/null +++ b/src/schemas/knowledge.js @@ -0,0 +1,97 @@ +const { serviceNameRegex } = require('./utils/utils') + +const knowledgeCreate = { + id: '/knowledgeCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: ['string', 'null'] + } + }, + required: ['name', 'repo', 'registryId'], + additionalProperties: false +} + +const knowledgeUpdate = { + id: '/knowledgeUpdate', + type: 'object', + properties: { + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: ['string', 'null'] + } + }, + additionalProperties: false +} + +const knowledgeLink = { + id: '/knowledgeLink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +const knowledgeUnlink = { + id: '/knowledgeUnlink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +module.exports = { + mainSchemas: [knowledgeCreate, knowledgeUpdate, knowledgeLink, knowledgeUnlink], + innerSchemas: [knowledgeCreate, knowledgeUpdate] +} diff --git a/src/services/event-service.js b/src/services/event-service.js index a862ca31..6eef2250 100644 --- a/src/services/event-service.js +++ b/src/services/event-service.js @@ -30,6 +30,7 @@ function extractResourceType (path) { { pattern: /^\/api\/v3\/registries/, type: 'registry' }, { pattern: /^\/api\/v3\/volumeMounts/, type: 'volumeMount' }, { pattern: /^\/api\/v3\/models/, type: 'model' }, + { pattern: /^\/api\/v3\/knowledge/, type: 'knowledge' }, { pattern: /^\/api\/v3\/runtimeClasses/, type: 'runtimeClass' }, { pattern: /^\/api\/v3\/microserviceTemplates/, type: 'microserviceTemplate' }, { pattern: /^\/api\/v3\/configMaps/, type: 'configMap' }, diff --git a/src/services/knowledge-service.js b/src/services/knowledge-service.js new file mode 100644 index 00000000..047befb0 --- /dev/null +++ b/src/services/knowledge-service.js @@ -0,0 +1,293 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const { normalizeKnowledgeFormat } = require('../helpers/knowledge-format') +const FleetKnowledgeManager = require('../data/managers/fleet-knowledge-manager') +const RegistryManager = require('../data/managers/registry-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const MicroserviceKnowledgeItemManager = require('../data/managers/microservice-knowledge-item-manager') +const ApplicationManager = require('../data/managers/application-manager') +const FogManager = require('../data/managers/iofog-manager') +const ChangeTrackingService = require('./change-tracking-service') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function toWireKnowledge (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + const item = { + uuid: plain.uuid, + name: plain.name, + repo: plain.repo, + revision: plain.revision != null ? plain.revision : '', + registryId: plain.registryId, + files: Array.isArray(plain.files) ? plain.files : [] + } + if (plain.format != null && plain.format !== '') { + item.format = plain.format + } + return item +} + +async function findLinkedFogUuids (knowledge, transaction) { + if (!knowledge || typeof knowledge.getFogs !== 'function') { + return [] + } + const fogs = await knowledge.getFogs({ transaction }) + return fogs.map((fog) => fog.uuid) +} + +async function _updateChangeTrackingForFogs (fogUuids, transaction) { + for (const fogUuid of fogUuids) { + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.knowledge, transaction) + } +} + +async function _isUserMicroservice (microservice, transaction) { + if (!microservice || microservice.isController) { + return false + } + if (!microservice.applicationId) { + return true + } + const application = await ApplicationManager.findOne({ id: microservice.applicationId }, transaction) + return !application || !application.isSystem +} + +async function findMicroservicesBindingKnowledgeName (knowledgeName, options, transaction) { + const items = await MicroserviceKnowledgeItemManager.findAll({ name: knowledgeName }, transaction) + const blocking = [] + const userOnly = Boolean(options && options.userOnly) + for (const item of items) { + if (userOnly || (options && options.fogUuid)) { + const microservice = await MicroserviceManager.findOne({ uuid: item.microserviceUuid }, transaction) + if (!microservice) { + continue + } + if (options && options.fogUuid && microservice.iofogUuid !== options.fogUuid) { + continue + } + if (userOnly && !(await _isUserMicroservice(microservice, transaction))) { + continue + } + blocking.push(microservice.uuid) + continue + } + blocking.push(item.microserviceUuid) + } + return blocking +} + +async function _validateRegistryAndFiles (registryId, files, transaction) { + const registry = await RegistryManager.findOne({ id: registryId }, transaction) + if (!registry) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryId)) + } + const type = registry.type || 'oci' + const fileList = Array.isArray(files) ? files : [] + if (type !== 'hf' && fileList.length > 0) { + throw new Errors.ValidationError(ErrorMessages.KNOWLEDGE_OCI_FILES_FORBIDDEN) + } + return registry +} + +async function getKnowledgeByName (name, transaction) { + const knowledge = await FleetKnowledgeManager.findOne({ name }, transaction) + if (!knowledge) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_NOT_FOUND, name)) + } + return knowledge +} + +async function listKnowledgeEndpoint (transaction) { + const knowledge = await FleetKnowledgeManager.findAll({}, transaction) + return { + knowledge: knowledge.map(toWireKnowledge) + } +} + +async function getKnowledgeEndpoint (name, transaction) { + const knowledge = await getKnowledgeByName(name, transaction) + return toWireKnowledge(knowledge) +} + +async function upsertKnowledgeEndpoint (name, data, transaction) { + const existing = await FleetKnowledgeManager.findOne({ name }, transaction) + if (existing) { + return updateKnowledgeEndpoint(name, data, transaction) + } + return createKnowledgeEndpoint({ ...data, name }, transaction) +} + +async function createKnowledgeEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.knowledgeCreate) + + const existing = await FleetKnowledgeManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_ALREADY_EXISTS, data.name)) + } + + const files = Array.isArray(data.files) ? data.files : [] + await _validateRegistryAndFiles(data.registryId, files, transaction) + + const created = await FleetKnowledgeManager.create({ + uuid: AppHelper.generateUUID(), + name: data.name, + repo: data.repo, + revision: data.revision != null ? data.revision : '', + registryId: data.registryId, + files, + format: normalizeKnowledgeFormat(data.format) + }, transaction) + + return toWireKnowledge(created) +} + +async function updateKnowledgeEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.knowledgeUpdate) + const knowledge = await getKnowledgeByName(name, transaction) + + const nextRepo = data.repo !== undefined ? data.repo : knowledge.repo + const nextRevision = data.revision !== undefined ? data.revision : (knowledge.revision != null ? knowledge.revision : '') + const nextRegistryId = data.registryId !== undefined ? data.registryId : knowledge.registryId + const nextFiles = data.files !== undefined ? data.files : (Array.isArray(knowledge.files) ? knowledge.files : []) + const nextFormat = data.format !== undefined ? normalizeKnowledgeFormat(data.format) : knowledge.format + + await _validateRegistryAndFiles(nextRegistryId, nextFiles, transaction) + + const linkedFogUuids = await findLinkedFogUuids(knowledge, transaction) + + await FleetKnowledgeManager.update({ name }, { + repo: nextRepo, + revision: nextRevision, + registryId: nextRegistryId, + files: nextFiles, + format: nextFormat + }, transaction) + + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + + return getKnowledgeEndpoint(name, transaction) +} + +async function deleteKnowledgeEndpoint (name, transaction) { + const knowledge = await getKnowledgeByName(name, transaction) + const blocking = await findMicroservicesBindingKnowledgeName(name, {}, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_IN_USE, name, blocking.join(', '))) + } + + const linkedFogUuids = await findLinkedFogUuids(knowledge, transaction) + await FleetKnowledgeManager.delete({ name }, transaction) + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + return {} +} + +async function getKnowledgeLinkEndpoint (name, transaction) { + const knowledge = await getKnowledgeByName(name, transaction) + return { + fogUuids: await findLinkedFogUuids(knowledge, transaction) + } +} + +async function linkKnowledgeEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.knowledgeLink) + const knowledge = await getKnowledgeByName(name, transaction) + const alreadyLinked = new Set(await findLinkedFogUuids(knowledge, transaction)) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + await agent.addKnowledge(knowledge, { transaction }) + } + + const newlyLinked = fogUuids.filter((uuid) => !alreadyLinked.has(uuid)) + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs(newlyLinked, transaction) + } + + return getKnowledgeEndpoint(name, transaction) +} + +async function unlinkKnowledgeEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.knowledgeUnlink) + const knowledge = await getKnowledgeByName(name, transaction) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + const blocking = await findMicroservicesBindingKnowledgeName(name, { fogUuid, userOnly: true }, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_UNLINK_IN_USE, name, fogUuid, blocking.join(', '))) + } + await agent.removeKnowledge(knowledge, { transaction }) + } + + await _updateChangeTrackingForFogs(fogUuids, transaction) + return {} +} + +async function ensureKnowledgeLinkedToFog (fogUuid, names, transaction) { + const knowledgeNames = [...new Set((names || []).filter((name) => typeof name === 'string' && name.length > 0))] + if (knowledgeNames.length === 0) { + return [] + } + + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + + const newlyLinked = [] + for (const name of knowledgeNames) { + const knowledge = await FleetKnowledgeManager.findOne({ name }, transaction) + if (!knowledge) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_NOT_FOUND, name)) + } + const alreadyLinked = typeof agent.hasKnowledge === 'function' + ? await agent.hasKnowledge(knowledge, { transaction }) + : false + if (alreadyLinked) { + continue + } + await agent.addKnowledge(knowledge, { transaction }) + newlyLinked.push(name) + } + + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs([fogUuid], transaction) + } + + return newlyLinked +} + +module.exports = { + listKnowledgeEndpoint: TransactionDecorator.generateTransaction(listKnowledgeEndpoint), + getKnowledgeEndpoint: TransactionDecorator.generateTransaction(getKnowledgeEndpoint), + createKnowledgeEndpoint: TransactionDecorator.generateTransaction(createKnowledgeEndpoint), + updateKnowledgeEndpoint: TransactionDecorator.generateTransaction(updateKnowledgeEndpoint), + deleteKnowledgeEndpoint: TransactionDecorator.generateTransaction(deleteKnowledgeEndpoint), + getKnowledgeLinkEndpoint: TransactionDecorator.generateTransaction(getKnowledgeLinkEndpoint), + linkKnowledgeEndpoint: TransactionDecorator.generateTransaction(linkKnowledgeEndpoint), + unlinkKnowledgeEndpoint: TransactionDecorator.generateTransaction(unlinkKnowledgeEndpoint), + upsertKnowledgeEndpoint: TransactionDecorator.generateTransaction(upsertKnowledgeEndpoint), + ensureKnowledgeLinkedToFog: TransactionDecorator.generateTransaction(ensureKnowledgeLinkedToFog), + toWireKnowledge +} diff --git a/src/services/yaml-parser-service.js b/src/services/yaml-parser-service.js index cfd33962..bd678ab5 100644 --- a/src/services/yaml-parser-service.js +++ b/src/services/yaml-parser-service.js @@ -412,6 +412,7 @@ const parseMicroserviceYAML = async (microservice) => { extraHosts: lget(microservice, 'container.extraHosts', []), application: microservice.application, models: microservice.models, + knowledge: microservice.knowledge, template: parseMicroserviceTemplateRef(microservice.template), schedule: lget(microservice, 'schedule', 50), serviceAccount: lget(microservice, 'serviceAccount', undefined), @@ -778,6 +779,55 @@ async function parseModelFile (fileContent, options = {}) { } } +async function parseKnowledgeFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'Knowledge') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null || doc.spec == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata or spec') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.knowledgeName) { + if (doc.metadata.name !== options.knowledgeName) { + throw new Errors.ValidationError(`Knowledge name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.knowledgeName})`) + } + } + + const spec = doc.spec || {} + const registryId = spec.registryId != null ? spec.registryId : spec.registry + const result = { + name: lget(doc, 'metadata.name', undefined), + repo: spec.repo, + revision: spec.revision, + registryId, + files: spec.files, + format: spec.format + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.knowledgeName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + async function parseRuntimeClassFile (fileContent, options = {}) { try { const doc = yaml.load(fileContent) @@ -861,6 +911,9 @@ async function parseMicroserviceTemplateYAML (microservice) { if (microservice.models !== undefined) { microserviceData.models = microservice.models } + if (microservice.knowledge !== undefined) { + microserviceData.knowledge = microservice.knowledge + } if (microservice.schedule !== undefined) { microserviceData.schedule = microservice.schedule } @@ -989,6 +1042,7 @@ module.exports = { parseRoleBindingFile, parseServiceAccountFile, parseModelFile, + parseKnowledgeFile, parseRuntimeClassFile, parseMicroserviceTemplateFile } From 430af080b3f28193599adca2e4cda6e02b8ed817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 00:04:37 +0300 Subject: [PATCH 43/55] Bind a knowledge catalog on user microservices and require items when bindPath is set. --- src/controllers/microservices-controller.js | 7 + src/helpers/microservice-container-catalog.js | 69 ++++++- src/routes/microservices.js | 36 ++++ src/schemas/microservice.js | 2 + src/services/microservices-service.js | 179 +++++++++++++++++- 5 files changed, 277 insertions(+), 16 deletions(-) diff --git a/src/controllers/microservices-controller.js b/src/controllers/microservices-controller.js index 704596cd..efa5bcc6 100644 --- a/src/controllers/microservices-controller.js +++ b/src/controllers/microservices-controller.js @@ -68,6 +68,12 @@ const updateMicroserviceCatalogEndPoint = async function (req) { return MicroservicesService.updateMicroserviceCatalogEndPoint(microserviceUuid, catalog, false) } +const updateMicroserviceKnowledgeEndPoint = async function (req) { + const microserviceUuid = req.params.uuid + const catalog = req.body + return MicroservicesService.updateMicroserviceKnowledgeEndPoint(microserviceUuid, catalog, false) +} + const updateMicroserviceConfigEndPoint = async function (req) { const microserviceUuid = req.params.uuid const config = req.body @@ -221,6 +227,7 @@ module.exports = { updateMicroserviceYAMLEndPoint, updateSystemMicroserviceYAMLEndPoint, updateMicroserviceCatalogEndPoint, + updateMicroserviceKnowledgeEndPoint, updateMicroserviceConfigEndPoint, getMicroserviceConfigEndPoint, updateSystemMicroserviceConfigEndPoint, diff --git a/src/helpers/microservice-container-catalog.js b/src/helpers/microservice-container-catalog.js index 3a815fe1..ab57c319 100644 --- a/src/helpers/microservice-container-catalog.js +++ b/src/helpers/microservice-container-catalog.js @@ -181,6 +181,45 @@ function collectMountPaths (volumeMappings, tmpfs) { return paths } +function catalogMountPaths (catalog) { + if (isCatalogEmpty(catalog)) { + return [] + } + const paths = [] + if (catalog.bindPath) { + paths.push(catalog.bindPath) + } + for (const item of catalog.items) { + if (item && item.name) { + paths.push(joinCatalogItemPath(catalog.bindPath, item.name)) + } + } + return paths +} + +function catalogValidationMessages (kind) { + if (kind === 'knowledge') { + return { + permissions: ErrorMessages.MICROSERVICE_KNOWLEDGE_PERMISSIONS, + bindRequired: ErrorMessages.MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRED, + bindRequiresItems: ErrorMessages.MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRES_ITEMS, + bindAbsolute: ErrorMessages.MICROSERVICE_KNOWLEDGE_BIND_PATH_ABSOLUTE, + itemName: ErrorMessages.MICROSERVICE_KNOWLEDGE_ITEM_NAME, + duplicate: ErrorMessages.MICROSERVICE_KNOWLEDGE_DUPLICATE_ITEM, + collision: ErrorMessages.MICROSERVICE_KNOWLEDGE_PATH_COLLISION + } + } + return { + permissions: ErrorMessages.MICROSERVICE_CATALOG_PERMISSIONS, + bindRequired: ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_REQUIRED, + bindRequiresItems: ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_REQUIRES_ITEMS, + bindAbsolute: ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE, + itemName: ErrorMessages.MICROSERVICE_CATALOG_ITEM_NAME, + duplicate: ErrorMessages.MICROSERVICE_CATALOG_DUPLICATE_ITEM, + collision: ErrorMessages.MICROSERVICE_CATALOG_PATH_COLLISION + } +} + function pathsCollide (left, right) { if (!left || !right) { return false @@ -190,42 +229,46 @@ function pathsCollide (left, right) { return a === b } -function validateCatalog (catalog, { volumeMappings, tmpfs } = {}) { +function validateCatalog (catalog, { volumeMappings, tmpfs, otherCatalog, kind } = {}) { if (catalog == null) { return } + const messages = catalogValidationMessages(kind) if (catalog.permissions != null && catalog.permissions !== 'ro' && catalog.permissions !== 'rw') { - throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_CATALOG_PERMISSIONS) + throw new Errors.ValidationError(messages.permissions) } const items = Array.isArray(catalog.items) ? catalog.items : [] if (items.length === 0) { + if (typeof catalog.bindPath === 'string' && catalog.bindPath.trim() !== '') { + throw new Errors.ValidationError(messages.bindRequiresItems) + } return } if (!catalog.bindPath) { - throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_REQUIRED) + throw new Errors.ValidationError(messages.bindRequired) } if (!isAbsoluteContainerPath(catalog.bindPath) && catalog.bindPath !== '/') { - throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE) + throw new Errors.ValidationError(messages.bindAbsolute) } const seen = new Set() for (const item of items) { const name = item && item.name if (!name || !SERVICE_NAME_RE.test(name)) { - throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_ITEM_NAME, name || '')) + throw new Errors.ValidationError(AppHelper.formatMessage(messages.itemName, name || '')) } if (seen.has(name)) { - throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_DUPLICATE_ITEM, name)) + throw new Errors.ValidationError(AppHelper.formatMessage(messages.duplicate, name)) } seen.add(name) } - const mountPaths = collectMountPaths(volumeMappings, tmpfs) + const mountPaths = collectMountPaths(volumeMappings, tmpfs).concat(catalogMountPaths(otherCatalog)) const catalogPaths = [catalog.bindPath, ...items.map((item) => joinCatalogItemPath(catalog.bindPath, item.name))] for (const catalogPath of catalogPaths) { for (const mountPath of mountPaths) { if (pathsCollide(catalogPath, mountPath)) { - throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_PATH_COLLISION, catalogPath)) + throw new Errors.ValidationError(AppHelper.formatMessage(messages.collision, catalogPath)) } } } @@ -483,8 +526,11 @@ function containerFieldsRequireRebuild (spec, existing) { spec.devices !== undefined || spec.tmpfs !== undefined) { return true } - if (spec.models !== undefined) { - return catalogRequiresRebuild(existing.models, normalizeCatalog(spec.models)) + if (spec.models !== undefined && catalogRequiresRebuild(existing.models, normalizeCatalog(spec.models))) { + return true + } + if (spec.knowledge !== undefined && catalogRequiresRebuild(existing.knowledge, normalizeCatalog(spec.knowledge))) { + return true } return false } @@ -601,6 +647,9 @@ function applyAgentContainerFields (response, microservice, extras = {}) { if (!isCatalogEmpty(extras.models)) { response.models = extras.models } + if (!isCatalogEmpty(extras.knowledge)) { + response.knowledge = extras.knowledge + } return response } diff --git a/src/routes/microservices.js b/src/routes/microservices.js index 9782126a..d45a68d8 100644 --- a/src/routes/microservices.js +++ b/src/routes/microservices.js @@ -297,6 +297,42 @@ module.exports = [ }) } }, + { + method: 'patch', + path: '/api/v3/microservices/:uuid/knowledge', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + + const successCode = constants.HTTP_CODE_NO_CONTENT + const errorCodes = [ + { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] + }, + { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] + }, + { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] + } + ] + + await rbacMiddleware.protect()(req, res, async () => { + const updateMicroserviceKnowledgeEndPoint = ResponseDecorator.handleErrors(MicroservicesController.updateMicroserviceKnowledgeEndPoint, + successCode, errorCodes) + const responseObject = await updateMicroserviceKnowledgeEndPoint(req) + const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' + res + .status(responseObject.code) + .send(responseObject.body) + + logger.apiRes({ req, user, res, responseObject }) + }) + } + }, { method: 'patch', path: '/api/v3/microservices/:uuid/rebuild', diff --git a/src/schemas/microservice.js b/src/schemas/microservice.js index 56aa5a47..a877b229 100644 --- a/src/schemas/microservice.js +++ b/src/schemas/microservice.js @@ -135,6 +135,7 @@ const microserviceCreate = { items: { $ref: '/containerTmpfs' } }, models: { $ref: '/microserviceCatalog' }, + knowledge: { $ref: '/microserviceCatalog' }, natsConfig: { $ref: '/microserviceNatsConfig' }, healthCheck: { type: 'object', @@ -289,6 +290,7 @@ const microserviceUpdate = { items: { $ref: '/containerTmpfs' } }, models: { $ref: '/microserviceCatalog' }, + knowledge: { $ref: '/microserviceCatalog' }, natsConfig: { $ref: '/microserviceNatsConfig' }, healthCheck: { type: 'object', diff --git a/src/services/microservices-service.js b/src/services/microservices-service.js index 13054846..8ac4d0bb 100644 --- a/src/services/microservices-service.js +++ b/src/services/microservices-service.js @@ -9,6 +9,8 @@ const MicroserviceTmpfsManager = require('../data/managers/microservice-tmpfs-ma const MicroserviceUlimitManager = require('../data/managers/microservice-ulimit-manager') const MicroserviceModelManager = require('../data/managers/microservice-model-manager') const MicroserviceModelItemManager = require('../data/managers/microservice-model-item-manager') +const MicroserviceKnowledgeManager = require('../data/managers/microservice-knowledge-manager') +const MicroserviceKnowledgeItemManager = require('../data/managers/microservice-knowledge-item-manager') const MicroserviceCdiDevManager = require('../data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../data/managers/microservice-cap-drop-manager') @@ -42,8 +44,10 @@ const SecretManager = require('../data/managers/secret-manager') const VolumeMountService = require('./volume-mount-service') const RuntimeClassService = require('./runtime-class-service') const ModelService = require('./model-service') +const KnowledgeService = require('./knowledge-service') const RegistryService = require('./registry-service') const FleetModelManager = require('../data/managers/fleet-model-manager') +const FleetKnowledgeManager = require('../data/managers/fleet-knowledge-manager') const CatalogContainer = require('../helpers/microservice-container-catalog') const RbacServiceAccountManager = require('../data/managers/rbac-service-account-manager') const RbacRoleManager = require('../data/managers/rbac-role-manager') @@ -502,24 +506,84 @@ async function _autoAttachCatalogModels (fogUuid, catalog, transaction) { await ModelService.ensureModelsLinkedToFog(fogUuid, names, transaction) } +async function _assertCatalogKnowledgeExist (catalog, transaction) { + const names = CatalogContainer.catalogItemNames(catalog) + for (const name of names) { + const knowledge = await FleetKnowledgeManager.findOne({ name }, transaction) + if (!knowledge) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_KNOWLEDGE_NOT_FOUND, name)) + } + } +} + +async function _autoAttachCatalogKnowledge (fogUuid, catalog, transaction) { + if (!fogUuid) { + return + } + const names = CatalogContainer.catalogItemNames(catalog) + if (names.length === 0) { + return + } + await KnowledgeService.ensureKnowledgeLinkedToFog(fogUuid, names, transaction) +} + +function _assertKnowledgeCatalogAllowed (microserviceData, microservice) { + if (!microserviceData || microserviceData.knowledge === undefined) { + return + } + const catalogItem = microservice && microservice.catalogItem + const isSystem = catalogItem && catalogItem.category === 'SYSTEM' + const isController = microservice && microservice.isController + if (isSystem || isController) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_KNOWLEDGE_SYSTEM_FORBIDDEN) + } +} + async function _loadCatalog (microserviceUuid, transaction) { const catalogRow = await MicroserviceModelManager.findOne({ microserviceUuid }, transaction) const itemRows = await MicroserviceModelItemManager.findAll({ microserviceUuid }, transaction) return CatalogContainer.catalogFromRows(catalogRow, itemRows) } +async function _loadKnowledgeCatalog (microserviceUuid, transaction) { + const catalogRow = await MicroserviceKnowledgeManager.findOne({ microserviceUuid }, transaction) + const itemRows = await MicroserviceKnowledgeItemManager.findAll({ microserviceUuid }, transaction) + return CatalogContainer.catalogFromRows(catalogRow, itemRows) +} + async function _loadTmpfs (microserviceUuid, transaction) { const rows = await MicroserviceTmpfsManager.findAll({ microserviceUuid }, transaction) return CatalogContainer.tmpfsFromRows(rows) } +function _catalogMountContext (microserviceData, existing, volumeMappings, tmpfs) { + return { + volumeMappings: volumeMappings !== undefined ? volumeMappings : existing.volumeMappings, + tmpfs: tmpfs !== undefined ? tmpfs : (microserviceData.tmpfs !== undefined ? microserviceData.tmpfs : existing.tmpfs) + } +} + function _prepareCatalogAndContainerFields (microserviceData, existing = {}, volumeMappings, tmpfs) { CatalogContainer.validateContainerFields(microserviceData, existing) + const mountContext = _catalogMountContext(microserviceData, existing, volumeMappings, tmpfs) + const nextModels = microserviceData.models !== undefined + ? CatalogContainer.normalizeCatalog(microserviceData.models) + : existing.models + const nextKnowledge = microserviceData.knowledge !== undefined + ? CatalogContainer.normalizeCatalog(microserviceData.knowledge) + : existing.knowledge if (microserviceData.models !== undefined) { - const catalog = CatalogContainer.normalizeCatalog(microserviceData.models) - CatalogContainer.validateCatalog(catalog, { - volumeMappings: volumeMappings !== undefined ? volumeMappings : existing.volumeMappings, - tmpfs: tmpfs !== undefined ? tmpfs : (microserviceData.tmpfs !== undefined ? microserviceData.tmpfs : existing.tmpfs) + CatalogContainer.validateCatalog(nextModels, { + ...mountContext, + otherCatalog: nextKnowledge, + kind: 'model' + }) + } + if (microserviceData.knowledge !== undefined) { + CatalogContainer.validateCatalog(nextKnowledge, { + ...mountContext, + otherCatalog: nextModels, + kind: 'knowledge' }) } } @@ -606,6 +670,26 @@ async function _replaceCatalog (catalogInput, microserviceUuid, transaction) { } } +async function _replaceKnowledgeCatalog (catalogInput, microserviceUuid, transaction) { + await MicroserviceKnowledgeItemManager.delete({ microserviceUuid }, transaction) + await MicroserviceKnowledgeManager.delete({ microserviceUuid }, transaction) + const catalog = CatalogContainer.normalizeCatalog(catalogInput) + if (CatalogContainer.isCatalogEmpty(catalog)) { + return + } + await MicroserviceKnowledgeManager.create({ + bindPath: catalog.bindPath, + permissions: catalog.permissions || 'ro', + microserviceUuid + }, transaction) + for (const item of catalog.items) { + await MicroserviceKnowledgeItemManager.create({ + name: item.name, + microserviceUuid + }, transaction) + } +} + async function _createContainerChildRows (microservice, microserviceData, transaction) { if (microserviceData.entrypoint !== undefined) { await _replaceEntrypoint(microserviceData.entrypoint, microservice.uuid, transaction) @@ -622,6 +706,9 @@ async function _createContainerChildRows (microservice, microserviceData, transa if (microserviceData.models !== undefined) { await _replaceCatalog(microserviceData.models, microservice.uuid, transaction) } + if (microserviceData.knowledge !== undefined) { + await _replaceKnowledgeCatalog(microserviceData.knowledge, microservice.uuid, transaction) + } } async function _applyCatalogContainerUpdate (microserviceData, microservice, microserviceDataUpdate, fogUuid, transaction) { @@ -632,7 +719,10 @@ async function _applyCatalogContainerUpdate (microserviceData, microservice, mic ? microserviceData.tmpfs : await _loadTmpfs(microservice.uuid, transaction) const existingCatalog = await _loadCatalog(microservice.uuid, transaction) + const existingKnowledge = await _loadKnowledgeCatalog(microservice.uuid, transaction) microservice.models = existingCatalog + microservice.knowledge = existingKnowledge + _assertKnowledgeCatalogAllowed(microserviceData, microservice) _prepareCatalogAndContainerFields(microserviceData, microservice, volumeMappings, tmpfs) _assignContainerColumns(microserviceDataUpdate, microserviceData) const argv = CatalogContainer.resolveProcessArgv(microserviceData) @@ -645,6 +735,11 @@ async function _applyCatalogContainerUpdate (microserviceData, microservice, mic await _assertCatalogModelsExist(catalog, transaction) await _autoAttachCatalogModels(fogUuid, catalog, transaction) } + if (microserviceData.knowledge !== undefined) { + const catalog = CatalogContainer.normalizeCatalog(microserviceData.knowledge) + await _assertCatalogKnowledgeExist(catalog, transaction) + await _autoAttachCatalogKnowledge(fogUuid, catalog, transaction) + } await _createContainerChildRows(microservice, microserviceData, transaction) } @@ -791,9 +886,10 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) await _validateMicroserviceRuntime(microserviceData.runtime, fog, transaction) // validate images + let catalogItem = null if (microserviceData.catalogItemId) { // validate catalog item - const catalogItem = await CatalogService.getCatalogItem(microserviceData.catalogItemId, isCLI, transaction) + catalogItem = await CatalogService.getCatalogItem(microserviceData.catalogItemId, isCLI, transaction) validateImagesAgainstCatalog(catalogItem, microserviceData.images || []) microserviceData.images = catalogItem.images _validateImageArch(microserviceData, fog, catalogItem.images) @@ -820,9 +916,13 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) _validateVolumeMappings(microserviceData.volumeMappings) _prepareCatalogAndContainerFields(microserviceData, {}, microserviceData.volumeMappings, microserviceData.tmpfs) + _assertKnowledgeCatalogAllowed(microserviceData, { catalogItem }) if (microserviceData.models !== undefined) { await _assertCatalogModelsExist(CatalogContainer.normalizeCatalog(microserviceData.models), transaction) } + if (microserviceData.knowledge !== undefined) { + await _assertCatalogKnowledgeExist(CatalogContainer.normalizeCatalog(microserviceData.knowledge), transaction) + } const microservice = await _createMicroservice({ ...microserviceData, iofogUuid: fog.uuid }, isCLI, transaction) @@ -902,6 +1002,9 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) if (microserviceData.models !== undefined) { await _autoAttachCatalogModels(fog.uuid, CatalogContainer.normalizeCatalog(microserviceData.models), transaction) } + if (microserviceData.knowledge !== undefined) { + await _autoAttachCatalogKnowledge(fog.uuid, CatalogContainer.normalizeCatalog(microserviceData.knowledge), transaction) + } if (microserviceData.iofogUuid) { // Full create always reloads the microservice list on the agent. @@ -1757,6 +1860,12 @@ async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, i if (changeTrackingEnabled) { await _updateChangeTracking(true, microservice.iofogUuid, transaction) await _updateChangeTracking(true, updatedMicroservice.iofogUuid, transaction) + if (microserviceData.knowledge !== undefined) { + await _flagMicroserviceKnowledge(microservice.iofogUuid, transaction) + if (updatedMicroservice.iofogUuid !== microservice.iofogUuid) { + await _flagMicroserviceKnowledge(updatedMicroservice.iofogUuid, transaction) + } + } } else { return { microserviceIofogUuid: microservice.iofogUuid, @@ -1810,9 +1919,12 @@ async function updateMicroserviceCatalogEndPoint (microserviceUuid, catalogData, const catalog = CatalogContainer.normalizeCatalog(catalogData) const volumeMappings = await VolumeMappingManager.findAll({ microserviceUuid }, transaction) const tmpfs = await _loadTmpfs(microserviceUuid, transaction) + const knowledgeCatalog = await _loadKnowledgeCatalog(microserviceUuid, transaction) CatalogContainer.validateCatalog(catalog, { volumeMappings, - tmpfs + tmpfs, + otherCatalog: knowledgeCatalog, + kind: 'model' }) await _assertCatalogModelsExist(catalog, transaction) await _autoAttachCatalogModels(microservice.iofogUuid, catalog, transaction) @@ -1836,6 +1948,51 @@ async function updateMicroserviceCatalogEndPoint (microserviceUuid, catalogData, } } +async function updateMicroserviceKnowledgeEndPoint (microserviceUuid, catalogData, isCLI, transaction) { + await Validator.validate(catalogData, Validator.schemas.microserviceCatalogPatch) + + const query = { uuid: microserviceUuid } + const microservice = await MicroserviceManager.findOneWithCategory(query, transaction) + if (!microservice) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, microserviceUuid)) + } + if (microservice.catalogItem && microservice.catalogItem.category === 'SYSTEM') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (microservice.isController) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (!isCLI) { + await _validateMicroserviceOnGet(microserviceUuid, transaction) + } + + const catalog = CatalogContainer.normalizeCatalog(catalogData) + const volumeMappings = await VolumeMappingManager.findAll({ microserviceUuid }, transaction) + const tmpfs = await _loadTmpfs(microserviceUuid, transaction) + const modelsCatalog = await _loadCatalog(microserviceUuid, transaction) + CatalogContainer.validateCatalog(catalog, { + volumeMappings, + tmpfs, + otherCatalog: modelsCatalog, + kind: 'knowledge' + }) + await _assertCatalogKnowledgeExist(catalog, transaction) + await _autoAttachCatalogKnowledge(microservice.iofogUuid, catalog, transaction) + + const previousCatalog = await _loadKnowledgeCatalog(microserviceUuid, transaction) + const rebuild = CatalogContainer.catalogRequiresRebuild(previousCatalog, catalog) + await _replaceKnowledgeCatalog(catalog, microserviceUuid, transaction) + await MicroserviceManager.update(query, { + rebuild: rebuild || !!microservice.rebuild + }, transaction) + + await _flagMicroserviceKnowledge(microservice.iofogUuid, transaction) + + return { + uuid: microserviceUuid + } +} + async function getMicroserviceConfigEndPoint (microserviceUuid, isCLI, transaction) { const query = isCLI ? { @@ -2876,6 +3033,13 @@ async function _updateSystemPorts (newPortMappings, microservice, transaction) { } } +async function _flagMicroserviceKnowledge (fogUuid, transaction) { + if (!fogUuid) { + return + } + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceKnowledge, transaction) +} + async function _updateChangeTracking (configUpdated, fogNodeUuid, transaction) { if (configUpdated) { // Spec updates other than a catalog-only PATCH reload the full microservice list. @@ -2966,6 +3130,7 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { const ulimitRows = await MicroserviceUlimitManager.findAll({ microserviceUuid }, transaction) const ulimits = CatalogContainer.ulimitsFromRows(ulimitRows) const catalog = await _loadCatalog(microserviceUuid, transaction) + const knowledge = await _loadKnowledgeCatalog(microserviceUuid, transaction) const cdiDevices = await MicroserviceCdiDevManager.findAllExcludeFields({ microserviceUuid }, transaction) const cdiDevs = cdiDevices.map((it) => it.cdiDevices) const capAdd = await MicroserviceCapAddManager.findAllExcludeFields({ microserviceUuid }, transaction) @@ -3002,6 +3167,7 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { res.devices = devices res.tmpfs = tmpfs res.models = catalog + res.knowledge = knowledge res.runAsGroup = res.runAsGroup || '' res.cdiDevices = cdiDevs res.capAdd = capAdds @@ -3194,6 +3360,7 @@ module.exports = { updateSystemMicroserviceEndPoint: TransactionDecorator.generateTransaction(updateSystemMicroserviceEndPoint), updateMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceConfigEndPoint), updateMicroserviceCatalogEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceCatalogEndPoint), + updateMicroserviceKnowledgeEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceKnowledgeEndPoint), getMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(getMicroserviceConfigEndPoint), getSystemMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(getSystemMicroserviceConfigEndPoint), deleteMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(deleteMicroserviceConfigEndPoint), From bcf7de0c4a768b14a5ec7c489811d1fcf081cb92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 00:04:41 +0300 Subject: [PATCH 44/55] Document fleet Knowledge in the API spec, access rules, and v3.9.0-rc.6 notes. --- CHANGELOG.md | 27 +- docs/architecture.md | 4 + docs/rbac-reference.md | 6 +- docs/swagger.yaml | 567 +++++++++++++++++++++++++++++++- src/config/rbac-resources.yaml | 35 ++ src/config/rbac-system-roles.js | 6 +- 6 files changed, 635 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebb60b78..01773779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,31 @@ # Changelog -## [v3.9.0-rc.4] +## [v3.9.0-rc.6] + +Fleet Knowledge artifacts: curated documents, datasets, and vector indexes distributed like Models, with a separate name namespace and Hub dataset pulls. + +### Added + +- **Fleet Knowledge API** — `GET/POST/PATCH/DELETE /api/v3/knowledge`, YAML create/upsert (`kind: Knowledge`), and volumeMount-style `GET/POST/DELETE …/knowledge/:name/link`; server-generated uuid; agent `GET /api/v3/agent/knowledge` returns linked rows only (`{ "knowledge": [ … ] }`). +- **`PATCH /api/v3/microservices/:uuid/knowledge`** — user microservices only; `microserviceKnowledge` change flag and rebuild rules for bind path / permissions / empty↔non-empty catalog transitions. +- **Fog status** — `knowledgeStatus` (JSON string), `activeKnowledge` (managed count), `knowledgeLastUpdate` (Unix ms). +- **v3.9.0 schema** — `Knowledge`, `FogKnowledge`, catalog tables, change-tracking flags, fog status columns on baseline and 3.8→3.9 upgrade. + +### Changed + +- **`modelLastUpdate`** — documented as Unix milliseconds (same clock as `knowledgeLastUpdate`). +- **MS catalog validation** — `knowledge.bindPath` must not collide with volumes, tmpfs, or the models catalog path. A non-empty `bindPath` on a models or knowledge catalog requires at least one item. +- **RBAC + swagger** — resource `knowledge`; viewer get/list; SRE/developer/admin full + link. +- Dockerfile base image digest pins refreshed for **`node:24-bookworm`** (console-builder and builder stages) +- Embedded **EdgeOps Console** default version **v1.1.0-rc.3** → **v1.1.0-rc.4** + +### Notes + +- Knowledge names are a separate namespace from Models. HF Knowledge uses the Hub dataset API; there is no `repoType` field. +- `format` is a soft hint. Known values persist; anything else stores `unknown`. Unknown format is not a 400. +- `getChanges.prune` still does not delete volumes. Unused **local** Knowledge prune is agent-side on the same flag. `system prune` does not drop Knowledge trees. + +## [v3.9.0-rc.5] Additive microservice `VOLUME` mapping `scope` for Edgelet private vs node-global shared claims. Reclaim stays on-node (`edgelet volume`); no new Controller prune flag. diff --git a/docs/architecture.md b/docs/architecture.md index ba6edaa6..40c64c9e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -358,6 +358,10 @@ Controller v3.8 and Edgelet share a **frozen field-agent REST contract** on `/ap **Volume mapping scope** — microservice `volumeMappings[]` may include `scope` (`private` default / `shared`). Meaningful only when `type` is `volume`. Shared names are node-global on that fog. Reclaim is on-node (`edgelet volume` / `deprovision --purge-volumes`); Controller does not prune volumes. `deleteWithCleanup` remains unused. +**Fleet models** — weight artifacts at `/api/v3/models`, linked per fog. Agent `GET /api/v3/agent/models` returns linked rows only. User microservices bind them with a `models` catalog (`bindPath`, `permissions`, `items[].name`). + +**Fleet Knowledge** — documents, datasets, and vector indexes, same distribution shape as models and a separate name namespace. Operator CRUD, YAML (`kind: Knowledge`), and per-fog link live at `/api/v3/knowledge`. User microservices bind a `knowledge` catalog and can change it with `PATCH /api/v3/microservices/:uuid/knowledge`. Agent `GET /api/v3/agent/knowledge` returns linked rows only. `format` is a soft hint. Hugging Face pulls use the dataset API. Fog `modelLastUpdate` and `knowledgeLastUpdate` are Unix milliseconds. + **Service account RBAC** — microservice roles use apiGroup **`edgelet.iofog.org/v1`**; SA/role changes trigger `microserviceList` change tracking. For the full bilateral contract (including ControlPlane env vars and verification references), see Edgelet documentation: diff --git a/docs/rbac-reference.md b/docs/rbac-reference.md index d9d60b21..7d54c2c8 100644 --- a/docs/rbac-reference.md +++ b/docs/rbac-reference.md @@ -1,7 +1,7 @@ # Controller RBAC — operator reference **Audience:** Platform operators, SREs, and integrators configuring access to Controller v3.8 -**Applies to:** User and admin HTTP APIs under `/api/v3/*` (not Edgelet agent wire protocol). Includes fleet models, RuntimeClass, and microservice templates. +**Applies to:** User and admin HTTP APIs under `/api/v3/*` (not Edgelet agent wire protocol). Includes fleet models, fleet Knowledge, RuntimeClass, and microservice templates. ## Overview @@ -86,7 +86,7 @@ The table below lists **user API** resources in `rbac-resources.yaml` and which | natsOperator, natsBootstrap, natsHub | read / ✓ | read | read | SRE has full NATS CRUD except operator objects (read) | | natsAccounts, natsUsers, natsAccountRules, natsUserRules | ✓ | ✓ | ✓ | | | catalog, registries | ✓ | ✓ | ✓ | | -| secrets, configMaps, volumeMounts, models, runtimeClasses | ✓ | ✓ | ✓ | Model/RuntimeClass link uses `patch` (same as volumeMounts) | +| secrets, configMaps, volumeMounts, models, knowledge, runtimeClasses | ✓ | ✓ | ✓ | Model, Knowledge, and RuntimeClass link uses `patch` (same as volumeMounts). Viewer is get/list only | | tunnels | ✓ | read | — | Viewer intentionally omits exec/tunnel paths | | certificates, capabilities | ✓ | ✓ | ✓ | | | execSessions, logs | ✓ | ✓ | — | | @@ -140,7 +140,7 @@ Some sub-resource actions map to `patch` (for example microservice start/stop). | `POST /api/v3/agent/controller/register` | Added under **`agent`** resource (fog token) | | OIDC auth routes (OAuth BFF, interactions) | Catalogued under **`users`** with auth-only verbs | -Fleet models, RuntimeClass, and microservice templates are user RBAC resources (`models`, `runtimeClasses`, `microserviceTemplates`). Viewer gets `get`/`list`. SRE, developer, and admin get full CRUD including link (`patch` on `…/:name/link`). Agent `GET /api/v3/agent/models` and `GET /api/v3/agent/runtimeClasses` are fog-token routes catalogued under **`agent`** for drift auditing only. +Fleet models, fleet Knowledge, RuntimeClass, and microservice templates are user RBAC resources (`models`, `knowledge`, `runtimeClasses`, `microserviceTemplates`). Viewer gets `get`/`list`. SRE, developer, and admin get full CRUD including link (`patch` on `…/:name/link`). Agent `GET /api/v3/agent/models`, `GET /api/v3/agent/knowledge`, and `GET /api/v3/agent/runtimeClasses` are fog-token routes catalogued under **`agent`** for drift auditing only. Catalog `PATCH /api/v3/microservices/:uuid/models` and `PATCH /api/v3/microservices/:uuid/knowledge` stay on the **`microservices`** resource. Removed user HAL/USB inventory routes (`GET /api/v3/iofog/:uuid/hal/hw`, `GET /api/v3/iofog/:uuid/hal/usb`) and agent HAL PUT routes have no yaml entries. diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 8aca045e..42ac8175 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1454,6 +1454,30 @@ paths: description: Not Authorized "500": description: Internal Server Error + /agent/knowledge: + get: + tags: + - Agent + summary: Gets fleet Knowledge linked to this fog + description: >- + Fog-token only. Returns Knowledge linked to the authenticated fog. + Rows missing uuid or name are omitted. An unlinked fog returns + { "knowledge": [] }. A missing route is treated as an empty list + by the agent. + operationId: getAgentLinkedKnowledge + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/AgentKnowledgeListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error /agent/runtimeClasses: get: tags: @@ -2725,6 +2749,46 @@ paths: description: Not Found "500": description: Internal Server Error + "/microservices/{uuid}/knowledge": + patch: + tags: + - Microservices + summary: Updates the microservice Knowledge catalog + description: >- + User microservices only. Catalog-only update. Sets the + microserviceKnowledge change flag only (does not set microserviceList). + Rebuilds the workload when the catalog goes empty to non-empty or + non-empty to empty, or when bindPath or permissions change. Adding or + removing items with the same bindPath and permissions does not rebuild + when the catalog is already non-empty. Catalog items reference managed + Knowledge names only. System and controller microservices are rejected. + operationId: patchMicroserviceKnowledge + parameters: + - in: path + name: uuid + description: Microservice Uuid + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeCatalog" + responses: + "204": + description: Catalog updated + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error "/microservices/{uuid}/rebuild": patch: tags: @@ -5844,6 +5908,338 @@ paths: "500": description: Internal Server Error + /knowledge: + get: + tags: + - Knowledge + summary: Returns fleet Knowledge + operationId: listKnowledge + security: + - authToken: [] + responses: + "200": + description: List of fleet Knowledge + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - Knowledge + summary: Creates fleet Knowledge + description: >- + Server generates uuid. Name is unique and immutable after create. + Names are a separate namespace from Models. Hugging Face pulls use + the dataset API. files is Hugging Face only; a non-empty files array on an + OCI registry is rejected. format is an optional soft hint (see + Knowledge.format). + operationId: createKnowledge + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /knowledge/yaml: + post: + tags: + - Knowledge + summary: Creates fleet Knowledge from YAML + description: >- + kind Knowledge. spec.registry is an alias of registryId. + apiVersion iofog.org/v3 or datasance.com/v3. + operationId: createKnowledgeYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + knowledge: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/knowledge/{name}": + get: + tags: + - Knowledge + summary: Gets fleet Knowledge + description: Does not include linked fog uuids. Use GET /knowledge/{name}/link. + operationId: getKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - Knowledge + summary: Updates fleet Knowledge + description: Name is immutable. format follows the same soft-hint rules as create. + operationId: updateKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Knowledge + summary: Deletes fleet Knowledge + description: Refused when any microservice binds the Knowledge name. + operationId: deleteKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Knowledge is bound by a microservice + "500": + description: Internal Server Error + + "/knowledge/yaml/{name}": + put: + tags: + - Knowledge + summary: Creates or updates fleet Knowledge from YAML + operationId: upsertKnowledgeYaml + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + knowledge: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + + "/knowledge/{name}/link": + get: + tags: + - Knowledge + summary: Gets fogs linked to fleet Knowledge + operationId: getKnowledgeLink + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + post: + tags: + - Knowledge + summary: Links fleet Knowledge to fog nodes + operationId: linkKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Knowledge + summary: Unlinks fleet Knowledge from fog nodes + description: Refused when a microservice on that fog still binds the Knowledge name. + operationId: unlinkKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Knowledge is bound by a microservice on the fog + "500": + description: Internal Server Error + /runtimeClasses: get: tags: @@ -8374,6 +8770,8 @@ tags: description: Manage your volume mounts - name: Models description: Manage your AI Model Catalogs + - name: Knowledge + description: Manage fleet Knowledge artifacts (documents, datasets, and vector indexes) - name: RuntimeClasses description: Manage your runtime classes - name: ConfigMap @@ -9052,7 +9450,20 @@ components: description: Count of managed fleet models only. modelLastUpdate: type: integer - description: Unix seconds; 0 when the managed list is empty. + description: Unix milliseconds; 0 when the managed list is empty. + knowledgeStatus: + type: string + description: >- + JSON string of local and managed Knowledge status items. Parse for + display. Managed rows include uuid; local rows omit uuid and set + source to local. source is local or managed. activeKnowledge is + the managed count, not the length of this array. + activeKnowledge: + type: integer + description: Count of managed fleet Knowledge only, not the knowledgeStatus row count. + knowledgeLastUpdate: + type: integer + description: Unix milliseconds; 0 when the managed list is empty. platformStatus: description: Present on GET single fog only; omitted on list responses allOf: @@ -9320,6 +9731,12 @@ components: microserviceModels: type: boolean description: Catalog-only refresh; agent uses GET microservices (no dedicated GET). + knowledge: + type: boolean + description: Replace managed Knowledge from GET /agent/knowledge. + microserviceKnowledge: + type: boolean + description: Catalog-only refresh; agent uses GET microservices (no dedicated GET). IOFogNodeConfig: type: object properties: @@ -9529,7 +9946,18 @@ components: description: Count of managed fleet models only. modelLastUpdate: type: integer - description: Unix seconds; 0 when the managed list is empty. + description: Unix milliseconds; 0 when the managed list is empty. + knowledgeStatus: + type: string + description: >- + JSON string of local and managed Knowledge status items. Managed + rows include uuid; local rows omit uuid. source is local or managed. + activeKnowledge: + type: integer + description: Count of managed fleet Knowledge only, not the knowledgeStatus row count. + knowledgeLastUpdate: + type: integer + description: Unix milliseconds; 0 when the managed list is empty. IOFogNodeTunnelConfigResponse: type: object properties: @@ -9650,6 +10078,11 @@ components: $ref: "#/components/schemas/ContainerTmpfs" models: $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. idConsumer: type: boolean isRouter: @@ -10286,6 +10719,11 @@ components: type: object models: $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. User microservices only. sysctls: type: object ulimits: @@ -10483,6 +10921,11 @@ components: $ref: "#/components/schemas/MicroserviceHealthCheck" models: $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. User microservices only. template: $ref: "#/components/schemas/MicroserviceTemplateRef" natsConfig: @@ -10614,6 +11057,11 @@ components: $ref: "#/components/schemas/MicroserviceHealthCheck" models: $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. User microservices only. template: $ref: "#/components/schemas/MicroserviceTemplateRef" natsConfig: @@ -12003,6 +12451,119 @@ components: - pytorch - tensorrt - unknown + Knowledge: + type: object + properties: + uuid: + type: string + name: + type: string + description: DNS-1123. Unique on the controller and immutable. Separate namespace from Models. + repo: + type: string + description: Hub dataset id or OCI path without host + revision: + type: string + description: Optional pin. Empty means latest (OCI) or main (HF). + registryId: + type: integer + description: Registry row. Hugging Face uses the dataset API. OCI uses ORAS. + files: + type: array + items: + type: string + description: Hugging Face only. Empty or omitted means the full dataset. Non-empty files on an OCI registry is rejected. + format: + type: string + description: >- + Optional soft hint. Known values persist lowercase: markdown, pdf, + jsonl, parquet, arrow, sqlite, faiss, chroma, lance. Any other + non-empty string is stored as unknown. Omit, null, or empty stays + omitted. Unknown format is not rejected. + KnowledgeListResponse: + type: object + properties: + knowledge: + type: array + items: + $ref: "#/components/schemas/Knowledge" + AgentKnowledgeListResponse: + type: object + properties: + knowledge: + type: array + items: + $ref: "#/components/schemas/Knowledge" + KnowledgeCreate: + type: object + required: + - name + - repo + - registryId + properties: + name: + type: string + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + description: >- + Optional soft hint. Known values persist lowercase: markdown, pdf, + jsonl, parquet, arrow, sqlite, faiss, chroma, lance. Any other + non-empty string is stored as unknown. Omit, null, or empty stays + omitted. Unknown format is not rejected. + KnowledgeUpdate: + type: object + properties: + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + description: >- + Optional soft hint. Known values persist lowercase: markdown, pdf, + jsonl, parquet, arrow, sqlite, faiss, chroma, lance. Any other + non-empty string is stored as unknown. Omit, null, or empty stays + omitted. Unknown format is not rejected. + KnowledgeCatalogItem: + type: object + required: + - name + properties: + name: + type: string + description: Managed fleet Knowledge name. Never uuid or a host path. + KnowledgeCatalog: + type: object + properties: + bindPath: + type: string + description: Required when items is non-empty. A non-empty bindPath requires at least one item. Absolute container path. Projection is {bindPath}/{name}/. + permissions: + type: string + enum: + - ro + - rw + default: ro + items: + type: array + items: + $ref: "#/components/schemas/KnowledgeCatalogItem" RuntimeClass: type: object properties: @@ -12052,7 +12613,7 @@ components: properties: bindPath: type: string - description: Required when items is non-empty. Absolute container path. + description: Required when items is non-empty. A non-empty bindPath requires at least one item. Absolute container path. permissions: type: string enum: diff --git a/src/config/rbac-resources.yaml b/src/config/rbac-resources.yaml index e56cfeb3..ef5e1663 100644 --- a/src/config/rbac-resources.yaml +++ b/src/config/rbac-resources.yaml @@ -59,6 +59,10 @@ resources: methods: PATCH: [patch] resourceNameParam: uuid + - path: /api/v3/microservices/:uuid/knowledge + methods: + PATCH: [patch] + resourceNameParam: uuid - path: /api/v3/microservices/:uuid/start methods: PATCH: [patch] @@ -553,6 +557,34 @@ resources: DELETE: [patch] resourceNameParam: name + # Fleet Knowledge + knowledge: + basePath: /api/v3/knowledge + routes: + - path: /api/v3/knowledge + methods: + GET: [list] + POST: [create] + - path: /api/v3/knowledge/yaml + methods: + POST: [create] + - path: /api/v3/knowledge/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/knowledge/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + - path: /api/v3/knowledge/:name/link + methods: + GET: [get] + POST: [patch] + DELETE: [patch] + resourceNameParam: name + # Runtime classes runtimeClasses: basePath: /api/v3/runtimeClasses @@ -721,6 +753,9 @@ resources: - path: /api/v3/agent/models methods: GET: [list] + - path: /api/v3/agent/knowledge + methods: + GET: [list] - path: /api/v3/agent/runtimeClasses methods: GET: [list] diff --git a/src/config/rbac-system-roles.js b/src/config/rbac-system-roles.js index 3fc0170d..58e457c2 100644 --- a/src/config/rbac-system-roles.js +++ b/src/config/rbac-system-roles.js @@ -40,7 +40,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'systemMicroservices', 'fogs', 'applications', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'runtimeClasses', 'tunnels', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'events', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'execSessions', 'systemExecSessions', 'logs', 'systemLogs'], + resources: ['microservices', 'systemMicroservices', 'fogs', 'applications', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'knowledge', 'runtimeClasses', 'tunnels', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'events', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'execSessions', 'systemExecSessions', 'logs', 'systemLogs'], verbs: ['*'] }, { @@ -62,7 +62,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'applications', 'applicationTemplates', 'microserviceTemplates', 'services', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'runtimeClasses', 'certificates', 'capabilities', 'serviceAccounts', 'controller', 'execSessions', 'logs'], + resources: ['microservices', 'applications', 'applicationTemplates', 'microserviceTemplates', 'services', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'knowledge', 'runtimeClasses', 'certificates', 'capabilities', 'serviceAccounts', 'controller', 'execSessions', 'logs'], verbs: ['get', 'list', 'create', 'update', 'patch', 'delete'] }, { @@ -84,7 +84,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'fogs', 'applications', 'systemMicroservices', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsOperator', 'natsBootstrap', 'natsHub', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'runtimeClasses', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'roles', 'roleBindings'], + resources: ['microservices', 'fogs', 'applications', 'systemMicroservices', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsOperator', 'natsBootstrap', 'natsHub', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'knowledge', 'runtimeClasses', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'roles', 'roleBindings'], verbs: ['get', 'list'] } ] From 25038dc81e98d2a8070ddc6c55628ee6c315c7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 00:04:47 +0300 Subject: [PATCH 45/55] Cover fleet Knowledge routes, catalogs, agent status, and access rules. --- test/src/config/rbac-audit.test.js | 15 +- test/src/controllers/agent-controller.test.js | 22 ++ .../microservice-container-catalog.test.js | 70 ++++ test/src/lib/rbac/authorizer.test.js | 24 ++ test/src/routes/agent-routes.test.js | 1 + test/src/routes/knowledge-routes.test.js | 32 ++ test/src/routes/microservices-routes.test.js | 3 + test/src/services/agent-service.test.js | 190 +++++++++++ .../services/change-tracking-service.test.js | 5 + test/src/services/iofog-service.test.js | 42 +++ test/src/services/knowledge-service.test.js | 303 ++++++++++++++++++ .../microservice-template-service.test.js | 11 + .../services/microservices-service.test.js | 296 +++++++++++++++++ test/src/services/yaml-parser-service.test.js | 122 +++++++ 14 files changed, 1133 insertions(+), 3 deletions(-) create mode 100644 test/src/routes/knowledge-routes.test.js create mode 100644 test/src/services/knowledge-service.test.js diff --git a/test/src/config/rbac-audit.test.js b/test/src/config/rbac-audit.test.js index d53c3be4..bb5384b3 100644 --- a/test/src/config/rbac-audit.test.js +++ b/test/src/config/rbac-audit.test.js @@ -16,28 +16,37 @@ describe('RBAC route catalog', () => { expect(result.missing).to.deep.equal([]) }) - it('registers fleet model, RuntimeClass, and microservice template routes', () => { + it('registers fleet model, Knowledge, RuntimeClass, and microservice template routes', () => { const catalog = yaml.load(fs.readFileSync(CATALOG_PATH, 'utf8')) expect(catalog.resources.models).to.be.an('object') + expect(catalog.resources.knowledge).to.be.an('object') expect(catalog.resources.runtimeClasses).to.be.an('object') expect(catalog.resources.microserviceTemplates).to.be.an('object') expect(catalog.resources.microservices.routes.some((route) => { return route.path === '/api/v3/microservices/:uuid/models' })).to.equal(true) + expect(catalog.resources.microservices.routes.some((route) => { + return route.path === '/api/v3/microservices/:uuid/knowledge' + })).to.equal(true) expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/models')).to.equal(true) + expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/knowledge')).to.equal(true) + expect(catalog.resources.knowledge.routes.some((route) => String(route.path).includes('/agent/'))).to.equal(false) expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/runtimeClasses')).to.equal(true) }) - it('does not leave stale yaml entries for fleet model or RuntimeClass routes', () => { + it('does not leave stale yaml entries for fleet model, Knowledge, or RuntimeClass routes', () => { const result = auditRbac() const fleetOrphans = result.orphans.filter((orphan) => { return orphan.resource === 'models' || + orphan.resource === 'knowledge' || orphan.resource === 'runtimeClasses' || orphan.resource === 'microserviceTemplates' || orphan.path.includes('/agent/models') || + orphan.path.includes('/agent/knowledge') || orphan.path.includes('/agent/runtimeClasses') || - orphan.path.includes('/microservices/:uuid/models') + orphan.path.includes('/microservices/:uuid/models') || + orphan.path.includes('/microservices/:uuid/knowledge') }) expect(fleetOrphans, JSON.stringify(fleetOrphans, null, 2)).to.deep.equal([]) }) diff --git a/test/src/controllers/agent-controller.test.js b/test/src/controllers/agent-controller.test.js index 7cdec596..46796702 100644 --- a/test/src/controllers/agent-controller.test.js +++ b/test/src/controllers/agent-controller.test.js @@ -548,6 +548,28 @@ describe('Agent Controller', () => { }) }) + describe('getAgentLinkedKnowledgeEndpoint()', () => { + def('fog', () => 'fog!') + def('req', () => ({ body: {} })) + def('response', () => Promise.resolve([])) + def('subject', () => $subject.getAgentLinkedKnowledgeEndpoint($req, $fog)) + + beforeEach(() => { + $sandbox.stub(AgentService, 'getAgentLinkedKnowledge').returns($response) + }) + + it('returns linked knowledge for the requesting fog', async () => { + await expect($subject).to.eventually.eql({ knowledge: [] }) + expect(AgentService.getAgentLinkedKnowledge).to.have.been.calledWith($fog) + }) + + it('keeps linked rows under the knowledge key', async () => { + const rows = [{ uuid: 'k-1', name: 'product-docs' }] + AgentService.getAgentLinkedKnowledge.resolves(rows) + await expect($subject).to.eventually.eql({ knowledge: rows }) + }) + }) + describe('getAgentLinkedRuntimeClassesEndpoint()', () => { def('fog', () => 'fog!') def('req', () => ({ body: {} })) diff --git a/test/src/helpers/microservice-container-catalog.test.js b/test/src/helpers/microservice-container-catalog.test.js index a38037e9..6e5f6e45 100644 --- a/test/src/helpers/microservice-container-catalog.test.js +++ b/test/src/helpers/microservice-container-catalog.test.js @@ -71,6 +71,27 @@ describe('Microservice container catalog helpers', () => { })).to.throw(Errors.ValidationError) }) + it('rejects a model bindPath when there are no items', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/models', + items: [] + })).to.throw(Errors.ValidationError, /requires at least one item/) + }) + + it('rejects a knowledge bindPath when items are omitted', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/knowledge' + }, { kind: 'knowledge' })).to.throw(Errors.ValidationError, /requires at least one item/) + }) + + it('accepts an empty catalog without bindPath', () => { + expect(() => CatalogContainer.validateCatalog({ items: [] })).to.not.throw() + expect(() => CatalogContainer.validateCatalog({ + bindPath: '', + items: [] + }, { kind: 'knowledge' })).to.not.throw() + }) + it('rejects duplicate item names', () => { expect(() => CatalogContainer.validateCatalog({ bindPath: '/models', @@ -86,6 +107,48 @@ describe('Microservice container catalog helpers', () => { volumeMappings: [{ containerDestination: '/data' }] })).to.throw(Errors.ValidationError) }) + + it('rejects a knowledge bindPath that matches the models catalog', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }, { + kind: 'knowledge', + otherCatalog: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + })).to.throw(Errors.ValidationError, /Knowledge catalog path/) + }) + + it('rejects a knowledge item path that matches a volume destination', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, { + kind: 'knowledge', + volumeMappings: [{ containerDestination: '/knowledge/product-docs' }] + })).to.throw(Errors.ValidationError, /Knowledge catalog path/) + }) + + it('rejects a knowledge bindPath that matches a tmpfs mount', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/run/knowledge', + items: [{ name: 'product-docs' }] + }, { + kind: 'knowledge', + tmpfs: [{ containerPath: '/run/knowledge' }] + })).to.throw(Errors.ValidationError, /Knowledge catalog path/) + }) + + it('rejects duplicate knowledge item names', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/knowledge', + items: [{ name: 'product-docs' }, { name: 'product-docs' }] + }, { kind: 'knowledge' })).to.throw(Errors.ValidationError, /duplicated/) + }) }) describe('.validateContainerFields()', () => { @@ -152,6 +215,11 @@ describe('Microservice container catalog helpers', () => { bindPath: '/models', permissions: 'ro', items: [{ name: 'test-model' }] + }, + knowledge: { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] } }) @@ -159,6 +227,8 @@ describe('Microservice container catalog helpers', () => { expect(response).to.not.have.property('commands') expect(response.cmd).to.eql([]) expect(response.models.items).to.eql([{ name: 'test-model' }]) + expect(response.knowledge.items).to.eql([{ name: 'product-docs' }]) + expect(response.knowledge.items[0]).to.not.have.property('uuid') }) it('emits commands from MicroserviceArgs rows', () => { diff --git a/test/src/lib/rbac/authorizer.test.js b/test/src/lib/rbac/authorizer.test.js index cda8343e..276f63fa 100644 --- a/test/src/lib/rbac/authorizer.test.js +++ b/test/src/lib/rbac/authorizer.test.js @@ -120,6 +120,30 @@ describe('RBAC authorizer fast path', () => { }) describe('authorize() full path', () => { + it('allows viewer get and list on knowledge and denies writes', async () => { + const getResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'get', null, $fakeTransaction + ) + const listResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'list', null, $fakeTransaction + ) + const createResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'create', null, $fakeTransaction + ) + const patchResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'patch', null, $fakeTransaction + ) + const deleteResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'delete', null, $fakeTransaction + ) + + expect(getResult.allowed).to.equal(true) + expect(listResult.allowed).to.equal(true) + expect(createResult.allowed).to.equal(false) + expect(patchResult.allowed).to.equal(false) + expect(deleteResult.allowed).to.equal(false) + }) + it('allows viewer system role for microservices get', async () => { const result = await authorizer.authorize( VIEWER_SUBJECTS, diff --git a/test/src/routes/agent-routes.test.js b/test/src/routes/agent-routes.test.js index 50e440b0..e164f500 100644 --- a/test/src/routes/agent-routes.test.js +++ b/test/src/routes/agent-routes.test.js @@ -8,6 +8,7 @@ describe('Agent routes', () => { it('registers fog-token model and runtime class list routes', () => { expect(source).to.include("path: '/api/v3/agent/models'") expect(source).to.include("path: '/api/v3/agent/runtimeClasses'") + expect(source).to.include("path: '/api/v3/agent/knowledge'") }) it('does not register agent hardware inventory routes', () => { diff --git a/test/src/routes/knowledge-routes.test.js b/test/src/routes/knowledge-routes.test.js new file mode 100644 index 00000000..4f58404d --- /dev/null +++ b/test/src/routes/knowledge-routes.test.js @@ -0,0 +1,32 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Knowledge routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/knowledge.js'), 'utf8') + + it('registers user knowledge CRUD, YAML, and link routes', () => { + expect(source).to.include("path: '/api/v3/knowledge'") + expect(source).to.include("path: '/api/v3/knowledge/:name'") + expect(source).to.include("path: '/api/v3/knowledge/yaml'") + expect(source).to.include("path: '/api/v3/knowledge/yaml/:name'") + expect(source).to.include("path: '/api/v3/knowledge/:name/link'") + expect(source).to.include("fileInput: 'knowledge'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + expect(source).to.not.include('/api/v3/knowledges') + }) + + it('documents knowledge and model status clocks as Unix milliseconds', () => { + const swagger = fs.readFileSync(path.resolve(__dirname, '../../../docs/swagger.yaml'), 'utf8') + for (const field of ['knowledgeLastUpdate:', 'modelLastUpdate:']) { + const blocks = swagger.split(field).slice(1) + expect(blocks.length).to.be.at.least(1) + for (const block of blocks) { + expect(block.slice(0, 500)).to.match(/Unix milliseconds/) + } + } + }) +}) diff --git a/test/src/routes/microservices-routes.test.js b/test/src/routes/microservices-routes.test.js index e6970b19..be56eca8 100644 --- a/test/src/routes/microservices-routes.test.js +++ b/test/src/routes/microservices-routes.test.js @@ -8,5 +8,8 @@ describe('Microservice routes', () => { it('registers catalog-only PATCH on a microservice', () => { expect(source).to.include("path: '/api/v3/microservices/:uuid/models'") expect(source).to.include('updateMicroserviceCatalogEndPoint') + expect(source).to.include("path: '/api/v3/microservices/:uuid/knowledge'") + expect(source).to.include('updateMicroserviceKnowledgeEndPoint') + expect(source).to.not.include("path: '/api/v3/microservices/system/:uuid/knowledge'") }) }) diff --git a/test/src/services/agent-service.test.js b/test/src/services/agent-service.test.js index 89e75ee7..ec39ec7c 100644 --- a/test/src/services/agent-service.test.js +++ b/test/src/services/agent-service.test.js @@ -582,6 +582,9 @@ describe('Agent Service', () => { expect(result.models).to.equal(false) expect(result.runtimeClasses).to.equal(false) expect(result.microserviceModels).to.equal(false) + expect(result.knowledge).to.equal(false) + expect(result.microserviceKnowledge).to.equal(false) + expect(result.prune).to.equal(false) }) context('when ChangeTrackingService#getByIoFogUuid() fails', () => { @@ -1437,9 +1440,26 @@ describe('Agent Service', () => { permissions: 'ro', items: [{ name: 'test-model' }] }) + expect(msvc).to.not.have.property('knowledge') expect(msvc.cpus).to.equal(1.5) expect(msvc.workingDir).to.equal('/app') }) + + it('emits knowledge item names and omits an empty catalog', async () => { + MicroserviceManager.findAllActiveApplicationMicroservices.resolves([{ + ...microserviceWithValidImage, + microserviceKnowledge: { bindPath: '/knowledge', permissions: 'ro' }, + knowledgeItems: [{ id: 1, name: 'product-docs' }] + }]) + + const result = await $subject + expect(result.microservices[0].knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) + expect(result.microservices[0].knowledge.items[0]).to.not.have.property('uuid') + }) }) }) }) @@ -1801,6 +1821,86 @@ describe('Agent Service', () => { }) }) + describe('.getAgentLinkedKnowledge()', () => { + const transaction = {} + + def('uuid', () => 'testUuid') + def('getKnowledge', () => $sandbox.stub().resolves([])) + def('fog', () => ({ + uuid: $uuid, + getKnowledge: $getKnowledge, + })) + def('subject', () => $subject.getAgentLinkedKnowledge($fog, transaction)) + + it('returns an empty list when the fog has no linked knowledge', async () => { + await expect($subject).to.eventually.eql([]) + expect($getKnowledge).to.have.been.calledWith({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction, + }) + }) + + it('maps linked knowledge and omits an unset format', async () => { + $getKnowledge.resolves([ + { + toJSON () { + return { + uuid: '3f2c1111-2222-3333-4444-555566667777', + name: 'product-docs', + repo: 'acme/product-manuals', + revision: null, + registryId: 3, + files: '["data/**/*.jsonl"]', + format: 'jsonl' + } + } + }, + { + toJSON () { + return { + uuid: '', + name: 'missing-uuid' + } + } + }, + { + toJSON () { + return { + uuid: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + name: 'wiki', + repo: 'acme/wiki', + revision: 'main', + registryId: 1, + files: ['index.md'], + format: '' + } + } + } + ]) + + await expect($subject).to.eventually.eql([ + { + uuid: '3f2c1111-2222-3333-4444-555566667777', + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }, + { + uuid: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + name: 'wiki', + repo: 'acme/wiki', + revision: 'main', + registryId: 1, + files: ['index.md'] + } + ]) + }) + }) + describe('.getAgentLinkedRuntimeClasses()', () => { const transaction = {} @@ -1913,6 +2013,96 @@ describe('Agent Service', () => { transaction ) }) + + it('stringifies a knowledge status array and keeps explicit zeros', async () => { + const status = { + ...agentStatus, + knowledgeStatus: [{ name: 'product-docs', state: 'Ready', source: 'managed' }], + activeKnowledge: 0, + knowledgeLastUpdate: 1710000000000 + } + await AgentService.updateAgentStatus(status, $fog, transaction) + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: $uuid }, + sinon.match({ + knowledgeStatus: JSON.stringify(status.knowledgeStatus), + activeKnowledge: 0, + knowledgeLastUpdate: 1710000000000 + }), + transaction + ) + }) + + it('leaves stored knowledge status unchanged when the keys are omitted', async () => { + await $subject + const update = ioFogManager.update.firstCall.args[1] + expect(update).to.not.have.property('knowledgeStatus') + expect(update).to.not.have.property('activeKnowledge') + expect(update).to.not.have.property('knowledgeLastUpdate') + }) + + it('stores the managed knowledge count separately from the status list length', async () => { + const status = { + ...agentStatus, + knowledgeStatus: [ + { name: 'product-docs', source: 'managed', uuid: '3f2c1111-2222-3333-4444-555566667777' }, + { name: 'local-notes', source: 'local' } + ], + activeKnowledge: 1, + knowledgeLastUpdate: 1710000000000 + } + await AgentService.updateAgentStatus(status, $fog, transaction) + const update = ioFogManager.update.firstCall.args[1] + expect(update.knowledgeStatus).to.equal(JSON.stringify(status.knowledgeStatus)) + expect(update.activeKnowledge).to.equal(1) + expect(JSON.parse(update.knowledgeStatus)).to.have.length(2) + expect(update.activeKnowledge).to.not.equal(JSON.parse(update.knowledgeStatus).length) + }) + + it('ignores unknown status fields', async () => { + const status = { + ...agentStatus, + knowledgeStatus: '[]', + activeKnowledge: 0, + knowledgeLastUpdate: 0, + futureKnowledgeHint: 'ignore-me' + } + await AgentService.updateAgentStatus(status, $fog, transaction) + const update = ioFogManager.update.firstCall.args[1] + expect(update).to.not.have.property('futureKnowledgeHint') + expect(update.knowledgeStatus).to.equal('[]') + expect(update.modelLastUpdate).to.equal(1710000000) + }) + }) + + describe('agent status schema', () => { + it('accepts a knowledge status string, an array, and omitted keys', async () => { + await expect(Validator.validate({ + knowledgeStatus: '[]', + activeKnowledge: 0, + knowledgeLastUpdate: 0 + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled + + await expect(Validator.validate({ + knowledgeStatus: [{ name: 'product-docs' }] + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled + + await expect(Validator.validate({ + daemonStatus: 'RUNNING' + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled + + await expect(Validator.validate({ + modelLastUpdate: 1710000000000, + knowledgeLastUpdate: 1710000000000, + unexpectedStatusExtra: true + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled + }) + + it('rejects a negative managed knowledge count', async () => { + await expect(Validator.validate({ + activeKnowledge: -1 + }, Validator.schemas.updateAgentStatus)).to.be.rejected + }) }) describe('.updateAgentStatus() observed runtime', () => { diff --git a/test/src/services/change-tracking-service.test.js b/test/src/services/change-tracking-service.test.js index a8d6459f..70bfdd1f 100644 --- a/test/src/services/change-tracking-service.test.js +++ b/test/src/services/change-tracking-service.test.js @@ -204,9 +204,14 @@ describe('ChangeTracking Service', () => { expect(ChangeTrackingService.events.clean.models).to.equal(false) expect(ChangeTrackingService.events.clean.runtimeClasses).to.equal(false) expect(ChangeTrackingService.events.clean.microserviceModels).to.equal(false) + expect(ChangeTrackingService.events.clean.knowledge).to.equal(false) + expect(ChangeTrackingService.events.clean.microserviceKnowledge).to.equal(false) expect(ChangeTrackingService.events.models).to.eql({ models: true }) expect(ChangeTrackingService.events.runtimeClasses).to.eql({ runtimeClasses: true }) expect(ChangeTrackingService.events.microserviceModels).to.eql({ microserviceModels: true }) + expect(ChangeTrackingService.events.knowledge).to.eql({ knowledge: true }) + expect(ChangeTrackingService.events.microserviceKnowledge).to.eql({ microserviceKnowledge: true }) + expect(ChangeTrackingService.events.prune).to.eql({ prune: true }) }) }) }) diff --git a/test/src/services/iofog-service.test.js b/test/src/services/iofog-service.test.js index 0fac5d3e..961d89b7 100644 --- a/test/src/services/iofog-service.test.js +++ b/test/src/services/iofog-service.test.js @@ -547,6 +547,48 @@ describe('ioFog Service', () => { expect(result).to.have.property('platformStatus', null) }) + context('when knowledge status is stored as a JSON string', () => { + beforeEach(() => { + ioFogManager.findOneWithTags.resolves(buildFogModel({ + uuid, + name: 'testName', + knowledgeStatus: JSON.stringify([{ name: 'product-docs', source: 'managed' }]), + activeKnowledge: 1, + knowledgeLastUpdate: 1710000000000 + })) + }) + + it('parses knowledgeStatus to an array on user GET', async () => { + const result = await $subject + expect(result.knowledgeStatus).to.eql([{ name: 'product-docs', source: 'managed' }]) + expect(result.activeKnowledge).to.equal(1) + expect(result.knowledgeLastUpdate).to.equal(1710000000000) + }) + }) + + context('when the status list is longer than the managed count', () => { + beforeEach(() => { + ioFogManager.findOneWithTags.resolves(buildFogModel({ + uuid, + name: 'testName', + knowledgeStatus: JSON.stringify([ + { name: 'product-docs', source: 'managed', uuid: '3f2c1111-2222-3333-4444-555566667777' }, + { name: 'local-notes', source: 'local' } + ]), + activeKnowledge: 1, + knowledgeLastUpdate: 1710000000000 + })) + }) + + it('keeps activeKnowledge as the managed count', async () => { + const result = await $subject + expect(result.knowledgeStatus).to.have.length(2) + expect(result.activeKnowledge).to.equal(1) + expect(result.activeKnowledge).to.not.equal(result.knowledgeStatus.length) + expect(result.knowledgeStatus[1]).to.not.have.property('uuid') + }) + }) + context('when platform status exists', () => { const lastTransitionAt = new Date('2026-06-24T12:00:00.000Z') diff --git a/test/src/services/knowledge-service.test.js b/test/src/services/knowledge-service.test.js new file mode 100644 index 00000000..b22e0fb4 --- /dev/null +++ b/test/src/services/knowledge-service.test.js @@ -0,0 +1,303 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const FleetKnowledgeManager = require('../../../src/data/managers/fleet-knowledge-manager') +const FleetModelManager = require('../../../src/data/managers/fleet-model-manager') +const RegistryManager = require('../../../src/data/managers/registry-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceKnowledgeItemManager = require('../../../src/data/managers/microservice-knowledge-item-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const KnowledgeService = require('../../../src/services/knowledge-service') +const { normalizeKnowledgeFormat } = require('../../../src/helpers/knowledge-format') +const AppHelper = require('../../../src/helpers/app-helper') +const Errors = require('../../../src/helpers/errors') + +describe('Knowledge Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + const knowledgeUuid = '3f2c1111-2222-3333-4444-555566667777' + const fogUuid = 'fog-1' + + afterEach(() => { + $sandbox.restore() + }) + + function stubKnowledgeRow (overrides = {}) { + return { + uuid: knowledgeUuid, + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl', + getFogs: $sandbox.stub().resolves([]), + ...overrides + } + } + + describe('format mapping', () => { + it('lowercases a known token and stores unknown for anything else', () => { + expect(normalizeKnowledgeFormat('JSONL')).to.equal('jsonl') + expect(normalizeKnowledgeFormat('PDF')).to.equal('pdf') + expect(normalizeKnowledgeFormat('docx')).to.equal('unknown') + expect(normalizeKnowledgeFormat('gguf')).to.equal('unknown') + expect(normalizeKnowledgeFormat('txt')).to.equal('unknown') + expect(normalizeKnowledgeFormat('Markdown ')).to.equal('unknown') + }) + + it('leaves an omitted, null, or empty format unset', () => { + expect(normalizeKnowledgeFormat(undefined)).to.equal(null) + expect(normalizeKnowledgeFormat(null)).to.equal(null) + expect(normalizeKnowledgeFormat('')).to.equal(null) + }) + }) + + describe('.createKnowledgeEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(AppHelper, 'generateUUID').returns(knowledgeUuid) + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(null) + $sandbox.stub(FleetKnowledgeManager, 'create').callsFake(async (data) => data) + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + }) + + it('persists a Hugging Face artifact with lowercase format and a server uuid', async () => { + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'JSONL' + }, transaction) + + expect(result).to.eql({ + uuid: knowledgeUuid, + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }) + expect(FleetKnowledgeManager.create).to.have.been.calledWith( + sinon.match({ uuid: knowledgeUuid, format: 'jsonl' }), + transaction + ) + }) + + it('stores unknown for an unrecognized format and does not reject the request', async () => { + for (const format of ['docx', 'gguf', 'txt']) { + FleetKnowledgeManager.create.resetHistory() + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + format + }, transaction) + expect(result.format).to.equal('unknown') + } + }) + + it('omits format when the field is left out', async () => { + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + files: ['data/**/*.jsonl'] + }, transaction) + + expect(result).to.not.have.property('format') + expect(FleetKnowledgeManager.create).to.have.been.calledWith( + sinon.match({ format: null }), + transaction + ) + }) + + it('rejects non-empty files on an OCI registry', async () => { + RegistryManager.findOne.resolves({ id: 1, type: 'oci' }) + + await expect(KnowledgeService.createKnowledgeEndpoint({ + name: 'oci-docs', + repo: 'acme/manuals', + registryId: 1, + files: ['index.md'] + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /must not specify files/) + expect(FleetKnowledgeManager.create).to.not.have.been.called + }) + + it('accepts an OCI registry when files are empty or omitted', async () => { + RegistryManager.findOne.resolves({ id: 1, type: 'oci' }) + + const withEmpty = await KnowledgeService.createKnowledgeEndpoint({ + name: 'oci-docs', + repo: 'acme/manuals', + registryId: 1, + files: [] + }, transaction) + expect(withEmpty.files).to.eql([]) + expect(withEmpty).to.not.have.property('format') + + const omitted = await KnowledgeService.createKnowledgeEndpoint({ + name: 'oci-docs', + repo: 'acme/manuals', + registryId: 1 + }, transaction) + expect(omitted.files).to.eql([]) + }) + + it('rejects a duplicate knowledge name', async () => { + FleetKnowledgeManager.findOne.resolves(stubKnowledgeRow()) + + await expect(KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3 + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + + it('allows a knowledge name that already belongs to a model', async () => { + $sandbox.stub(FleetModelManager, 'findOne').resolves({ name: 'product-docs', uuid: 'model-uuid' }) + + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }, transaction) + + expect(result.name).to.equal('product-docs') + expect(result.uuid).to.equal(knowledgeUuid) + expect(FleetModelManager.findOne).to.not.have.been.called + expect(FleetKnowledgeManager.create).to.have.been.calledOnce + }) + }) + + describe('.getKnowledgeEndpoint() and .getKnowledgeLinkEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(stubKnowledgeRow({ + getFogs: $sandbox.stub().resolves([{ uuid: fogUuid }]), + fogUuids: [fogUuid] + })) + }) + + it('returns the artifact spec without linked fogs', async () => { + const row = await FleetKnowledgeManager.findOne() + const result = await KnowledgeService.getKnowledgeEndpoint('product-docs', transaction) + expect(result).to.eql({ + uuid: knowledgeUuid, + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }) + expect(result).to.not.have.property('fogUuids') + expect(result).to.not.have.property('fogs') + expect(row.getFogs).to.not.have.been.called + }) + + it('returns linked fog uuids from the link endpoint', async () => { + const result = await KnowledgeService.getKnowledgeLinkEndpoint('product-docs', transaction) + expect(result).to.eql({ fogUuids: [fogUuid] }) + }) + }) + + describe('.updateKnowledgeEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(stubKnowledgeRow()) + $sandbox.stub(FleetKnowledgeManager, 'update').resolves() + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('rejects a name change because the name is immutable', async () => { + await expect(KnowledgeService.updateKnowledgeEndpoint('product-docs', { + name: 'renamed-docs', + repo: 'acme/other' + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /not allowed/) + expect(FleetKnowledgeManager.update).to.not.have.been.called + }) + + it('updates spec fields without writing the name', async () => { + await KnowledgeService.updateKnowledgeEndpoint('product-docs', { + revision: 'main', + files: ['index/faiss.index'] + }, transaction) + + const patch = FleetKnowledgeManager.update.firstCall.args[1] + expect(patch).to.not.have.property('name') + expect(patch.revision).to.equal('main') + expect(FleetKnowledgeManager.update.firstCall.args[0]).to.eql({ name: 'product-docs' }) + }) + }) + + describe('.deleteKnowledgeEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(stubKnowledgeRow()) + $sandbox.stub(FleetKnowledgeManager, 'delete').resolves() + $sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('refuses delete while a microservice binds the name and names that microservice', async () => { + MicroserviceKnowledgeItemManager.findAll.resolves([ + { microserviceUuid: 'ms-1', name: 'product-docs' } + ]) + + await expect(KnowledgeService.deleteKnowledgeEndpoint('product-docs', transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-1/) + expect(FleetKnowledgeManager.delete).to.not.have.been.called + }) + }) + + describe('link and unlink', () => { + let knowledge + let agent + + beforeEach(() => { + knowledge = stubKnowledgeRow() + agent = { + uuid: fogUuid, + addKnowledge: $sandbox.stub().resolves(), + removeKnowledge: $sandbox.stub().resolves() + } + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(knowledge) + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(MicroserviceManager, 'findOne') + $sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links a fog and sets the knowledge change flag', async () => { + const result = await KnowledgeService.linkKnowledgeEndpoint('product-docs', [fogUuid], transaction) + expect(agent.addKnowledge).to.have.been.calledWith(knowledge, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.knowledge, + transaction + ) + expect(result.name).to.equal('product-docs') + expect(result).to.not.have.property('fogUuids') + }) + + it('refuses unlink while a microservice on that fog binds the name', async () => { + MicroserviceKnowledgeItemManager.findAll.resolves([ + { microserviceUuid: 'ms-bound', name: 'product-docs' } + ]) + MicroserviceManager.findOne.resolves({ + uuid: 'ms-bound', + iofogUuid: fogUuid + }) + + await expect(KnowledgeService.unlinkKnowledgeEndpoint('product-docs', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-bound/) + expect(agent.removeKnowledge).to.not.have.been.called + expect(ChangeTrackingService.update).to.not.have.been.called + }) + }) +}) diff --git a/test/src/services/microservice-template-service.test.js b/test/src/services/microservice-template-service.test.js index 1250e345..39e87aa3 100644 --- a/test/src/services/microservice-template-service.test.js +++ b/test/src/services/microservice-template-service.test.js @@ -26,6 +26,11 @@ describe('Microservice Template Service', () => { bindPath: '/models', permissions: 'ro', items: [{ name: 'test-model' }] + }, + knowledge: { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] } } } @@ -72,6 +77,7 @@ describe('Microservice Template Service', () => { expect(storedMicroservice.agentName).to.equal('{{ agent-name }}') expect(storedMicroservice).to.not.have.property('iofogUuid') expect(storedMicroservice.models.items).to.eql([{ name: 'test-model' }]) + expect(storedMicroservice.knowledge.items).to.eql([{ name: 'product-docs' }]) expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) expect(result.variables[0].key).to.equal('port') expect(result.variables[0].defaultValue).to.equal('80') @@ -109,6 +115,11 @@ describe('Microservice Template Service', () => { expect(result.microserviceTemplates).to.have.length(1) expect(result.microserviceTemplates[0].name).to.equal('nginx-edge') expect(result.microserviceTemplates[0].microservice.models.bindPath).to.equal('/models') + expect(result.microserviceTemplates[0].microservice.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) expect(result.microserviceTemplates[0].variables[0].defaultValue).to.equal('80') }) diff --git a/test/src/services/microservices-service.test.js b/test/src/services/microservices-service.test.js index ec174d16..aaee3219 100644 --- a/test/src/services/microservices-service.test.js +++ b/test/src/services/microservices-service.test.js @@ -24,6 +24,8 @@ const MicroserviceTmpfsManager = require('../../../src/data/managers/microservic const MicroserviceUlimitManager = require('../../../src/data/managers/microservice-ulimit-manager') const MicroserviceModelManager = require('../../../src/data/managers/microservice-model-manager') const MicroserviceModelItemManager = require('../../../src/data/managers/microservice-model-item-manager') +const MicroserviceKnowledgeManager = require('../../../src/data/managers/microservice-knowledge-manager') +const MicroserviceKnowledgeItemManager = require('../../../src/data/managers/microservice-knowledge-item-manager') const MicroserviceCdiDevManager = require('../../../src/data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../../../src/data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../../../src/data/managers/microservice-cap-drop-manager') @@ -31,8 +33,10 @@ const MicroserviceHealthCheckManager = require('../../../src/data/managers/micro const VolumeMountService = require('../../../src/services/volume-mount-service') const RuntimeClassService = require('../../../src/services/runtime-class-service') const ModelService = require('../../../src/services/model-service') +const KnowledgeService = require('../../../src/services/knowledge-service') const MicroserviceTemplateService = require('../../../src/services/microservice-template-service') const FleetModelManager = require('../../../src/data/managers/fleet-model-manager') +const FleetKnowledgeManager = require('../../../src/data/managers/fleet-knowledge-manager') const RbacRoleManager = require('../../../src/data/managers/rbac-role-manager') const ErrorMessages = require('../../../src/helpers/error-messages') const RbacServiceAccountManager = require('../../../src/data/managers/rbac-service-account-manager') @@ -81,6 +85,12 @@ function stubChildTableDeps (sandbox) { sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) sandbox.stub(MicroserviceModelItemManager, 'create').resolves() sandbox.stub(MicroserviceModelItemManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceKnowledgeManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceKnowledgeItemManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'delete').resolves() } function stubBuildGetResponseDeps (sandbox) { @@ -1501,6 +1511,63 @@ describe('Microservices Service', () => { expect(created).to.not.have.property('commands') }) + it('persists a knowledge catalog and auto-attaches names to the target agent', async () => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves({ uuid: 'knowledge-uuid', name: 'product-docs' }) + const attach = $sandbox.stub(KnowledgeService, 'ensureKnowledgeLinkedToFog').resolves(['product-docs']) + + await $service.createMicroserviceEndPoint({ + ...microserviceData, + knowledge: { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + } + }, isCLI, transaction) + + expect(attach).to.have.been.calledWith('fog-uuid', ['product-docs'], transaction) + expect(MicroserviceKnowledgeManager.create).to.have.been.calledWith( + sinon.match({ bindPath: '/knowledge', permissions: 'ro' }), + transaction + ) + expect(MicroserviceKnowledgeItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'product-docs' }), + transaction + ) + const created = MicroserviceManager.create.firstCall.args[0] + expect(created).to.not.have.property('knowledge') + }) + + it('rejects an unknown knowledge catalog item', async () => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(null) + + return expect($service.createMicroserviceEndPoint({ + ...microserviceData, + knowledge: { + bindPath: '/knowledge', + items: [{ name: 'missing-docs' }] + } + }, isCLI, transaction)).to.be.rejectedWith( + AppHelper.formatMessage(ErrorMessages.MICROSERVICE_KNOWLEDGE_NOT_FOUND, 'missing-docs') + ) + }) + + it('rejects knowledge on a system catalog microservice', async () => { + const CatalogService = require('../../../src/services/catalog-service') + $sandbox.stub(CatalogService, 'getCatalogItem').resolves({ + category: 'SYSTEM', + images: microserviceData.images, + registryId: 1 + }) + + return expect($service.createMicroserviceEndPoint({ + ...microserviceData, + catalogItemId: 9, + knowledge: { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + } + }, isCLI, transaction)).to.be.rejectedWith(ErrorMessages.MICROSERVICE_KNOWLEDGE_SYSTEM_FORBIDDEN) + }) + it('persists commands when cmd is sent as an alias', async () => { await $service.createMicroserviceEndPoint({ ...microserviceData, @@ -1602,6 +1669,12 @@ describe('Microservices Service', () => { sandbox.stub(MicroserviceModelManager, 'create').resolves() sandbox.stub(MicroserviceModelItemManager, 'delete').resolves() sandbox.stub(MicroserviceModelItemManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceKnowledgeManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'create').resolves() } it('rebuilds when the catalog goes from empty to non-empty', async () => { @@ -1715,6 +1788,216 @@ describe('Microservices Service', () => { transaction ) }) + + it('rejects a model bindPath that matches the knowledge catalog', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + MicroserviceKnowledgeManager.findOne.resolves({ bindPath: '/models', permissions: 'ro' }) + MicroserviceKnowledgeItemManager.findAll.resolves([{ id: 1, name: 'product-docs' }]) + + return expect($service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError) + }) + }) + + describe('.updateMicroserviceKnowledgeEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const existingCatalog = { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + } + + function stubKnowledgePatch (sandbox, existing, previousCatalog = null) { + sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(existing) + sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) + sandbox.stub(FleetKnowledgeManager, 'findOne').resolves({ uuid: 'knowledge-uuid', name: 'product-docs' }) + sandbox.stub(KnowledgeService, 'ensureKnowledgeLinkedToFog').resolves([]) + sandbox.stub(MicroserviceManager, 'update').resolves() + sandbox.stub(ChangeTrackingService, 'update').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceModelManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceKnowledgeManager, 'findOne').resolves( + previousCatalog && previousCatalog.bindPath + ? { bindPath: previousCatalog.bindPath, permissions: previousCatalog.permissions || 'ro' } + : null + ) + sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves( + previousCatalog && Array.isArray(previousCatalog.items) + ? previousCatalog.items.map((item, id) => ({ id, name: item.name })) + : [] + ) + sandbox.stub(MicroserviceKnowledgeManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'create').resolves() + } + + it('rebuilds when the catalog goes from empty to non-empty and flags knowledge only', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when the last item is removed', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + items: [] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(MicroserviceKnowledgeManager.create).to.not.have.been.called + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + }) + + it('does not rebuild when only items change on a non-empty catalog', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + FleetKnowledgeManager.findOne.callsFake(async (where) => ({ uuid: 'knowledge-uuid', name: where.name })) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }, { name: 'wiki-faiss' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: false }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + expect(KnowledgeService.ensureKnowledgeLinkedToFog).to.have.been.calledWith( + existing.iofogUuid, + ['product-docs', 'wiki-faiss'], + transaction + ) + }) + + it('rebuilds when bindPath changes and flags knowledge only', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/data', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when permissions change and flags knowledge only', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + permissions: 'rw', + items: [{ name: 'product-docs' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rejects a system microservice', async () => { + stubKnowledgePatch($sandbox, buildMicroserviceRecord({ + catalogItem: { category: 'SYSTEM' } + })) + + return expect($service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError) + }) + + it('rejects a controller microservice', async () => { + stubKnowledgePatch($sandbox, buildMicroserviceRecord({ isController: true })) + + return expect($service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError) + }) + + it('returns not found when the microservice is missing', async () => { + $sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(null) + + return expect($service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + items: [] + }, isCLI, transaction)).to.be.rejectedWith(Errors.NotFoundError) + }) }) describe('.buildGetMicroserviceResponse() observed status', () => { @@ -1731,6 +2014,19 @@ describe('Microservices Service', () => { $sandbox.stub(ApplicationManager, 'findOne').resolves({ name: 'my-app' }) }) + it('returns the knowledge catalog with item names only', async () => { + MicroserviceKnowledgeManager.findOne.resolves({ bindPath: '/knowledge', permissions: 'rw' }) + MicroserviceKnowledgeItemManager.findAll.resolves([{ id: 2, name: 'product-docs' }]) + + const result = await $subject + expect(result.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'rw', + items: [{ name: 'product-docs' }] + }) + expect(result.knowledge.items[0]).to.not.have.property('uuid') + }) + it('zeros CPU when status is STOPPED', async () => { MicroserviceStatusManager.findAllExcludeFields.resolves([{ status: 'STOPPED', diff --git a/test/src/services/yaml-parser-service.test.js b/test/src/services/yaml-parser-service.test.js index 2e03294c..fd9e5f15 100644 --- a/test/src/services/yaml-parser-service.test.js +++ b/test/src/services/yaml-parser-service.test.js @@ -48,6 +48,30 @@ spec: } }) }) + + it('copies knowledge from an inline microservice', async () => { + const yaml = ` +kind: Application +metadata: + name: app-a +spec: + microservices: + - name: ms-a + knowledge: + bindPath: /knowledge + permissions: rw + items: + - name: product-docs + container: + env: [] +` + const result = await YamlParserService.parseAppFile(yaml) + expect(result.microservices[0].knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'rw', + items: [{ name: 'product-docs' }] + }) + }) }) describe('.parseMicroserviceFile()', () => { @@ -165,6 +189,28 @@ spec: expect(result).to.not.have.property('volumes') }) + it('maps spec.knowledge next to models', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-knowledge +spec: + knowledge: + bindPath: /knowledge + permissions: ro + items: + - name: product-docs + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) + }) + it('maps container devices, tmpfs, sysctls, and ulimits', async () => { const yaml = ` kind: Microservice @@ -419,6 +465,72 @@ spec: }) }) + describe('.parseKnowledgeFile()', () => { + it('parses kind Knowledge into the wire object', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: Knowledge +metadata: + name: product-docs +spec: + repo: acme/product-manuals + revision: abc123 + registryId: 3 + files: + - data/**/*.jsonl + format: JSONL +` + const result = await YamlParserService.parseKnowledgeFile(yaml) + expect(result).to.eql({ + name: 'product-docs', + repo: 'acme/product-manuals', + revision: 'abc123', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'JSONL' + }) + }) + + it('accepts spec.registry as an alias for registryId', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: Knowledge +metadata: + name: product-docs +spec: + repo: acme/product-manuals + registry: 3 + files: + - data/**/*.jsonl +` + const result = await YamlParserService.parseKnowledgeFile(yaml) + expect(result.registryId).to.equal(3) + expect(result).to.not.have.property('labels') + }) + + it('omits name on upsert and rejects a path mismatch', async () => { + const yaml = ` +kind: Knowledge +metadata: + name: product-docs +spec: + repo: acme/product-manuals + registryId: 3 +` + const updated = await YamlParserService.parseKnowledgeFile(yaml, { + isUpdate: true, + knowledgeName: 'product-docs' + }) + expect(updated).to.not.have.property('name') + expect(updated.repo).to.equal('acme/product-manuals') + + await expect(YamlParserService.parseKnowledgeFile(yaml, { + isUpdate: true, + knowledgeName: 'other-docs' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) + describe('.parseRuntimeClassFile()', () => { it('parses metadata.name and top-level handler matching the wire object', async () => { const yaml = ` @@ -511,6 +623,11 @@ spec: permissions: ro items: - name: test-model + knowledge: + bindPath: /knowledge + permissions: ro + items: + - name: product-docs container: commands: - nginx @@ -534,6 +651,11 @@ spec: permissions: 'ro', items: [{ name: 'test-model' }] }) + expect(result.microservice.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) expect(result.microservice.env).to.eql([{ key: 'PORT', value: '{{ port }}' }]) expect(result.microservice).to.not.have.property('name') From 956af6462c917970a4e760a53f7184ffb1bfaffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 00:04:54 +0300 Subject: [PATCH 46/55] Bump the embedded EdgeOps Console default to v1.1.0-rc.4. --- .env.example | 2 +- .github/actions/set-build-env/action.yml | 2 +- Dockerfile | 8 ++++---- Makefile | 4 ++-- scripts/build-console-dev.js | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index e7c45fd8..a787bed9 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 +EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index b71f5c90..c3d02f54 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.3"; fi + if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.4"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/Dockerfile b/Dockerfile index 14733d94..e3d08ada 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ # ioFog overrides: EDGEOPS_CONSOLE_REPO=https://github.com/eclipse-iofog/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=iofog # node:24-bookworm — pin manifest list digest for reproducible multi-arch builds -FROM node:24-bookworm@sha256:6dac556d980b7f0e5498d08f08cee0ca67798b4ad6c23964a9214920e67758d0 AS console-builder +FROM node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -28,7 +28,7 @@ RUN test -f build/index.html \ && cp -a build /tmp/console/build -FROM node:24-bookworm@sha256:6dac556d980b7f0e5498d08f08cee0ca67798b4ad6c23964a9214920e67758d0 AS builder +FROM node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 AS builder ARG PKG_VERSION @@ -50,7 +50,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:d0adaa87e9c71479064b9455a1bd778a26e36ccb53be9e6bc73eae98e2be3bf6 -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index 63ae0040..dceeb832 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.3 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.3 +EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.4 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index a8188d49..56ca80e5 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.3' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.4' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { From fddae8b6d7e7ac3bda75b208c3a1400245ae74a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 13:16:18 +0300 Subject: [PATCH 47/55] Return fog knowledgeStatus as a JSON string so it matches modelStatus, and refresh the UBI Node base digest. --- CHANGELOG.md | 7 +++++++ Dockerfile | 2 +- src/services/iofog-service.js | 18 ----------------- test/src/services/iofog-service.test.js | 27 ++++++++++++++++--------- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01773779..f478998e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [v3.9.0-rc.7] + +### Changed + +- **Fog status** — user GET and list return `knowledgeStatus` as the stored JSON string, matching `modelStatus`. Callers parse it. +- Dockerfile runtime base image digest pin refreshed for **`ubi9/nodejs-24-minimal`**. + ## [v3.9.0-rc.6] Fleet Knowledge artifacts: curated documents, datasets, and vector indexes distributed like Models, with a separate name namespace and Hub dataset pulls. diff --git a/Dockerfile b/Dockerfile index e3d08ada..76e1df4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds -FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:d0adaa87e9c71479064b9455a1bd778a26e36ccb53be9e6bc73eae98e2be3bf6 +FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:e3392c129520d2ebce748c3d0048343d0ec60be560cf220b60a7ba5d2bd2484a ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 ARG IMAGE_REGISTRY diff --git a/src/services/iofog-service.js b/src/services/iofog-service.js index 037df383..c602d6a1 100644 --- a/src/services/iofog-service.js +++ b/src/services/iofog-service.js @@ -825,21 +825,6 @@ async function _getFogVolumeMounts (fog, transaction) { }) } -function _parseKnowledgeStatus (value) { - if (typeof value !== 'string') { - return Array.isArray(value) ? value : value - } - if (value === '') { - return [] - } - try { - const parsed = JSON.parse(value) - return Array.isArray(parsed) ? parsed : [] - } catch (e) { - return [] - } -} - async function _getFogExtraInformation (fog, transaction, options = {}) { const fogUuid = fog.uuid const parsedSpec = await FogPlatformSpecManager.getParsedSpec(fogUuid, transaction) @@ -861,9 +846,6 @@ async function _getFogExtraInformation (fog, transaction, options = {}) { } : undefined const result = { ...fogFields, archId, arch, tags: _mapTags(fog), ...routerConfig, ...natsConfig, volumeMounts } - if (Object.prototype.hasOwnProperty.call(result, 'knowledgeStatus')) { - result.knowledgeStatus = _parseKnowledgeStatus(result.knowledgeStatus) - } if (options.includePlatformStatus) { const status = await FogPlatformStatusManager.getParsedStatus(fogUuid, transaction) result.platformStatus = _formatPlatformStatus(status, parsedSpec ? parsedSpec.generation : null) diff --git a/test/src/services/iofog-service.test.js b/test/src/services/iofog-service.test.js index 961d89b7..d7b6b54e 100644 --- a/test/src/services/iofog-service.test.js +++ b/test/src/services/iofog-service.test.js @@ -548,33 +548,38 @@ describe('ioFog Service', () => { }) context('when knowledge status is stored as a JSON string', () => { + const knowledgeStatusJson = JSON.stringify([{ name: 'product-docs', source: 'managed' }]) + beforeEach(() => { ioFogManager.findOneWithTags.resolves(buildFogModel({ uuid, name: 'testName', - knowledgeStatus: JSON.stringify([{ name: 'product-docs', source: 'managed' }]), + knowledgeStatus: knowledgeStatusJson, activeKnowledge: 1, knowledgeLastUpdate: 1710000000000 })) }) - it('parses knowledgeStatus to an array on user GET', async () => { + it('returns knowledgeStatus as the stored JSON string on user GET', async () => { const result = await $subject - expect(result.knowledgeStatus).to.eql([{ name: 'product-docs', source: 'managed' }]) + expect(result.knowledgeStatus).to.equal(knowledgeStatusJson) + expect(JSON.parse(result.knowledgeStatus)).to.eql([{ name: 'product-docs', source: 'managed' }]) expect(result.activeKnowledge).to.equal(1) expect(result.knowledgeLastUpdate).to.equal(1710000000000) }) }) context('when the status list is longer than the managed count', () => { + const knowledgeStatusJson = JSON.stringify([ + { name: 'product-docs', source: 'managed', uuid: '3f2c1111-2222-3333-4444-555566667777' }, + { name: 'local-notes', source: 'local' } + ]) + beforeEach(() => { ioFogManager.findOneWithTags.resolves(buildFogModel({ uuid, name: 'testName', - knowledgeStatus: JSON.stringify([ - { name: 'product-docs', source: 'managed', uuid: '3f2c1111-2222-3333-4444-555566667777' }, - { name: 'local-notes', source: 'local' } - ]), + knowledgeStatus: knowledgeStatusJson, activeKnowledge: 1, knowledgeLastUpdate: 1710000000000 })) @@ -582,10 +587,12 @@ describe('ioFog Service', () => { it('keeps activeKnowledge as the managed count', async () => { const result = await $subject - expect(result.knowledgeStatus).to.have.length(2) + const rows = JSON.parse(result.knowledgeStatus) + expect(result.knowledgeStatus).to.equal(knowledgeStatusJson) + expect(rows).to.have.length(2) expect(result.activeKnowledge).to.equal(1) - expect(result.activeKnowledge).to.not.equal(result.knowledgeStatus.length) - expect(result.knowledgeStatus[1]).to.not.have.property('uuid') + expect(result.activeKnowledge).to.not.equal(rows.length) + expect(rows[1]).to.not.have.property('uuid') }) }) From f702eec08f9a0213ad7494e18f013fd8e460a0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Tue, 22 Sep 2026 14:23:14 +0300 Subject: [PATCH 48/55] bump embedded console version to v1.1.0-rc.5 --- .env.example | 2 +- .github/actions/set-build-env/action.yml | 2 +- CHANGELOG.md | 5 +++++ Dockerfile | 4 ++-- Makefile | 4 ++-- scripts/build-console-dev.js | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index a787bed9..70dcde88 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 +EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index c3d02f54..c6d045dd 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.4"; fi + if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.5"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index f478998e..803e8ee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [v3.9.0-rc.8] + +### Changed +- Embedded **EdgeOps Console** default version **v1.1.0-rc.4** → **v1.1.0-rc.5** + ## [v3.9.0-rc.7] ### Changed diff --git a/Dockerfile b/Dockerfile index 76e1df4f..9a1591bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -50,7 +50,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:e3392c129520d2ebce748c3d0048343d0ec60be560cf220b60a7ba5d2bd2484a -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index dceeb832..6ffabfaf 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.4 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.4 +EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.5 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index 56ca80e5..2f0dac8a 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.4' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.5' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { From 3b9c59c8934e11f309fe8f849f167e889f92d70f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 23 Sep 2026 15:40:24 +0300 Subject: [PATCH 49/55] Add fog columns for host capacity and OS identity. Extend 3.9.0 upgrade and baseline migrations on sqlite, mysql, and postgres, and map the new fields on the Fog Sequelize model. --- .../db_migration_mysql_baseline_v3.9.0.sql | 6 ++++ .../mysql/db_migration_mysql_v3.9.0.sql | 7 ++++ .../db_migration_pg_baseline_v3.9.0.sql | 6 ++++ .../postgres/db_migration_pg_v3.9.0.sql | 7 ++++ .../db_migration_sqlite_baseline_v3.9.0.sql | 6 ++++ .../sqlite/db_migration_sqlite_v3.9.0.sql | 7 ++++ src/data/models/fog.js | 33 +++++++++++++++++++ 7 files changed, 72 insertions(+) diff --git a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql index 780894bf..72ed58db 100644 --- a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql @@ -73,6 +73,12 @@ CREATE TABLE IF NOT EXISTS Fogs ( system_available_disk BIGINT, system_available_memory BIGINT, system_total_cpu FLOAT, + system_cpus BIGINT, + system_total_memory BIGINT, + system_total_disk BIGINT, + system_os TEXT, + system_os_version TEXT, + system_kernel_version TEXT, security_status VARCHAR(36) DEFAULT 'OK', security_violation_info VARCHAR(36) DEFAULT 'No violation', catalog_item_status TEXT, diff --git a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql index c307d22c..66a8536a 100644 --- a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql @@ -233,3 +233,10 @@ ALTER TABLE ChangeTrackings ADD COLUMN microservice_knowledge BOOLEAN DEFAULT fa ALTER TABLE Fogs ADD COLUMN knowledge_status TEXT; ALTER TABLE Fogs ADD COLUMN active_knowledge BIGINT DEFAULT 0; ALTER TABLE Fogs ADD COLUMN knowledge_last_update BIGINT DEFAULT 0; + +ALTER TABLE Fogs ADD COLUMN system_cpus BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_memory BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_disk BIGINT; +ALTER TABLE Fogs ADD COLUMN system_os TEXT; +ALTER TABLE Fogs ADD COLUMN system_os_version TEXT; +ALTER TABLE Fogs ADD COLUMN system_kernel_version TEXT; diff --git a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql index 64bb3ec2..742c5d39 100644 --- a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql @@ -71,6 +71,12 @@ CREATE TABLE IF NOT EXISTS "Fogs" ( system_available_disk BIGINT, system_available_memory BIGINT, system_total_cpu DOUBLE PRECISION, + system_cpus BIGINT, + system_total_memory BIGINT, + system_total_disk BIGINT, + system_os TEXT, + system_os_version TEXT, + system_kernel_version TEXT, security_status VARCHAR(36) DEFAULT 'OK', security_violation_info VARCHAR(36) DEFAULT 'No violation', catalog_item_status TEXT, diff --git a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql index d1f2ffe9..fb3e494b 100644 --- a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql @@ -229,3 +229,10 @@ ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS microservice_knowledge BO ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS knowledge_status TEXT; ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS active_knowledge BIGINT DEFAULT 0; ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS knowledge_last_update BIGINT DEFAULT 0; + +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_cpus BIGINT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_total_memory BIGINT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_total_disk BIGINT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_os TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_os_version TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_kernel_version TEXT; diff --git a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql index cc38a04f..dd439212 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql @@ -71,6 +71,12 @@ CREATE TABLE IF NOT EXISTS Fogs ( system_available_disk BIGINT, system_available_memory BIGINT, system_total_cpu FLOAT, + system_cpus BIGINT, + system_total_memory BIGINT, + system_total_disk BIGINT, + system_os TEXT, + system_os_version TEXT, + system_kernel_version TEXT, security_status VARCHAR(36) DEFAULT 'OK', security_violation_info VARCHAR(36) DEFAULT 'No violation', catalog_item_status TEXT, diff --git a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql index 420751ba..9393fb36 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql @@ -246,3 +246,10 @@ ALTER TABLE ChangeTrackings ADD COLUMN microservice_knowledge BOOLEAN DEFAULT fa ALTER TABLE Fogs ADD COLUMN knowledge_status TEXT; ALTER TABLE Fogs ADD COLUMN active_knowledge BIGINT DEFAULT 0; ALTER TABLE Fogs ADD COLUMN knowledge_last_update BIGINT DEFAULT 0; + +ALTER TABLE Fogs ADD COLUMN system_cpus BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_memory BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_disk BIGINT; +ALTER TABLE Fogs ADD COLUMN system_os TEXT; +ALTER TABLE Fogs ADD COLUMN system_os_version TEXT; +ALTER TABLE Fogs ADD COLUMN system_kernel_version TEXT; diff --git a/src/data/models/fog.js b/src/data/models/fog.js index ca8ddadd..c3490867 100644 --- a/src/data/models/fog.js +++ b/src/data/models/fog.js @@ -118,6 +118,39 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.FLOAT, field: 'system_total_cpu' }, + systemCpus: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('systemCpus')) + }, + field: 'system_cpus' + }, + systemTotalMemory: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('systemTotalMemory')) + }, + field: 'system_total_memory' + }, + systemTotalDisk: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('systemTotalDisk')) + }, + field: 'system_total_disk' + }, + systemOs: { + type: DataTypes.TEXT, + field: 'system_os' + }, + systemOsVersion: { + type: DataTypes.TEXT, + field: 'system_os_version' + }, + systemKernelVersion: { + type: DataTypes.TEXT, + field: 'system_kernel_version' + }, securityStatus: { type: DataTypes.TEXT, defaultValue: 'OK', From b6805bb806b62359832036655ce5b1ee4aabee6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 23 Sep 2026 15:40:38 +0300 Subject: [PATCH 50/55] Persist host capacity and OS fields from agent status updates. Validate new status keys on PUT /api/v3/agent/status and map them into fog storage; add a unit test for the update path. --- src/schemas/agent.js | 8 +++++++- src/services/agent-service.js | 6 ++++++ test/src/services/agent-service.test.js | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/schemas/agent.js b/src/schemas/agent.js index af4bcdbe..7decec2b 100644 --- a/src/schemas/agent.js +++ b/src/schemas/agent.js @@ -31,7 +31,7 @@ const updateAgentConfig = { diskLimit: { type: 'integer', minimum: 0 }, diskDirectory: { type: 'string' }, memoryLimit: { type: 'integer', minimum: 0 }, - cpuLimit: { type: 'integer', minimum: 0 }, + cpuLimit: { type: 'integer', minimum: 5, maximum: 400 }, logLimit: { type: 'integer', minimum: 0 }, logDirectory: { type: 'string' }, logFileCount: { type: 'integer', minimum: 0 }, @@ -79,6 +79,12 @@ const updateAgentStatus = { systemAvailableDisk: { type: 'integer' }, systemAvailableMemory: { type: 'integer' }, systemTotalCpu: { type: 'number' }, + systemCpus: { type: 'integer', minimum: 0 }, + systemTotalMemory: { type: 'integer', minimum: 0 }, + systemTotalDisk: { type: 'integer', minimum: 0 }, + systemOs: { type: 'string' }, + systemOsVersion: { type: 'string' }, + systemKernelVersion: { type: 'string' }, securityStatus: { type: 'string' }, securityViolationInfo: { type: 'string' }, microserviceStatus: { type: 'string' }, diff --git a/src/services/agent-service.js b/src/services/agent-service.js index 5af9f135..a934247d 100644 --- a/src/services/agent-service.js +++ b/src/services/agent-service.js @@ -282,6 +282,12 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { systemAvailableDisk: agentStatus.systemAvailableDisk, systemAvailableMemory: agentStatus.systemAvailableMemory, systemTotalCpu: agentStatus.systemTotalCpu, + systemCpus: agentStatus.systemCpus, + systemTotalMemory: agentStatus.systemTotalMemory, + systemTotalDisk: agentStatus.systemTotalDisk, + systemOs: agentStatus.systemOs, + systemOsVersion: agentStatus.systemOsVersion, + systemKernelVersion: agentStatus.systemKernelVersion, securityStatus: agentStatus.securityStatus, securityViolationInfo: agentStatus.securityViolationInfo, repositoryCount: agentStatus.repositoryCount, diff --git a/test/src/services/agent-service.test.js b/test/src/services/agent-service.test.js index ec39ec7c..9c920355 100644 --- a/test/src/services/agent-service.test.js +++ b/test/src/services/agent-service.test.js @@ -2014,6 +2014,31 @@ describe('Agent Service', () => { ) }) + it('persists host capacity and OS identity from agent status', async () => { + const status = { + ...agentStatus, + systemCpus: 8, + systemTotalMemory: 17179869184, + systemTotalDisk: 494384795648, + systemOs: 'linux', + systemOsVersion: 'Ubuntu 22.04', + systemKernelVersion: '6.8.0-45-generic' + } + await AgentService.updateAgentStatus(status, $fog, transaction) + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: $uuid }, + sinon.match({ + systemCpus: 8, + systemTotalMemory: 17179869184, + systemTotalDisk: 494384795648, + systemOs: 'linux', + systemOsVersion: 'Ubuntu 22.04', + systemKernelVersion: '6.8.0-45-generic' + }), + transaction + ) + }) + it('stringifies a knowledge status array and keeps explicit zeros', async () => { const status = { ...agentStatus, From e7efae387688acff3ea911956174c87f8db88dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 23 Sep 2026 15:40:50 +0300 Subject: [PATCH 51/55] Document fog host status fields and usage semantics in OpenAPI. Clarify diskUsage versus host disk totals and systemTotalCpu as busy percent, and describe capacity and OS identity fields on agent status and user fog responses. --- docs/swagger.yaml | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 42ac8175..709e0f25 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -9316,6 +9316,7 @@ components: type: number diskUsage: type: number + description: Edgelet data-directory usage in GiB, not host filesystem total. cpuUsage: type: number memoryViolation: @@ -9324,8 +9325,33 @@ components: type: boolean cpuViolation: type: boolean + systemAvailableDisk: + type: integer + description: Free space on the agent diskDirectory filesystem in bytes. + systemAvailableMemory: + type: integer + description: Host RAM available in bytes. systemTotalCpu: type: number + description: Host CPU busy percentage 0–100, not logical core count. + systemCpus: + type: integer + description: Host logical CPU count. + systemTotalMemory: + type: integer + description: Host RAM capacity in bytes. + systemTotalDisk: + type: integer + description: diskDirectory filesystem size in bytes. + systemOs: + type: string + description: Host OS family (linux, darwin, windows, …). + systemOsVersion: + type: string + description: Human-oriented distro or release; empty string when unknown. + systemKernelVersion: + type: string + description: Linux kernel release; empty string on non-Linux. catalogItemStatus: type: string repositoryStatus: @@ -9354,6 +9380,8 @@ components: type: number cpuLimit: type: number + minimum: 5 + maximum: 400 logLimit: type: number logDirectory: @@ -9543,6 +9571,8 @@ components: cpuLimit: type: number default: 80 + minimum: 5 + maximum: 400 format: percent logLimit: type: number @@ -9760,6 +9790,8 @@ components: type: number cpuLimit: type: number + minimum: 5 + maximum: 400 logLimit: type: number logDirectory: @@ -9809,6 +9841,8 @@ components: type: number cpuLimit: type: number + minimum: 5 + maximum: 400 logLimit: type: number logDirectory: @@ -9855,6 +9889,7 @@ components: type: number diskUsage: type: number + description: Edgelet data-directory usage in GiB, not host filesystem total. cpuUsage: type: number memoryViolation: @@ -9865,10 +9900,31 @@ components: type: boolean systemAvailableDisk: type: integer + description: Free space on the agent diskDirectory filesystem in bytes. systemAvailableMemory: type: integer + description: Host RAM available in bytes. systemTotalCpu: type: number + description: Host CPU busy percentage 0–100, not logical core count. + systemCpus: + type: integer + description: Host logical CPU count. + systemTotalMemory: + type: integer + description: Host RAM capacity in bytes. + systemTotalDisk: + type: integer + description: diskDirectory filesystem size in bytes. + systemOs: + type: string + description: Host OS family (linux, darwin, windows, …). + systemOsVersion: + type: string + description: Human-oriented distro or release; empty string when unknown. + systemKernelVersion: + type: string + description: Linux kernel release; empty string on non-Linux. securityStatus: type: string securityViolationInfo: From 9488025876b8620493beb6357945418f55249ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 23 Sep 2026 15:40:56 +0300 Subject: [PATCH 52/55] Enforce cpuLimit between 5 and 400 percent on fog and agent config. Align JSON Schema validation and swagger with Edgelet percent limits for create, update, and agent config payloads. --- src/schemas/iofog.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/schemas/iofog.js b/src/schemas/iofog.js index c4dab556..de6ef7e5 100644 --- a/src/schemas/iofog.js +++ b/src/schemas/iofog.js @@ -16,7 +16,7 @@ const iofogCreate = { diskLimit: { type: 'integer', minimum: 0 }, diskDirectory: { type: 'string' }, memoryLimit: { type: 'integer', minimum: 0 }, - cpuLimit: { type: 'integer', minimum: 0 }, + cpuLimit: { type: 'integer', minimum: 5, maximum: 400 }, logLimit: { type: 'integer', minimum: 0 }, logDirectory: { type: 'string' }, logFileCount: { type: 'integer', minimum: 0 }, @@ -90,7 +90,7 @@ const iofogUpdate = { diskLimit: { type: 'integer', minimum: 0 }, diskDirectory: { type: 'string' }, memoryLimit: { type: 'integer', minimum: 0 }, - cpuLimit: { type: 'integer', minimum: 0 }, + cpuLimit: { type: 'integer', minimum: 5, maximum: 400 }, logLimit: { type: 'integer', minimum: 0 }, logDirectory: { type: 'string' }, logFileCount: { type: 'integer', minimum: 0 }, From 39272566573cf2111d7c0eec7d12f74c128ebf57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 23 Sep 2026 15:41:10 +0300 Subject: [PATCH 53/55] Note host capacity and OS status in the v3.9.0-rc.8 changelog. Document agent ingest, user GET/list exposure, and migration coverage without changing package version. --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 803e8ee2..ffaa0d1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,12 @@ ## [v3.9.0-rc.8] +### Added + +- **Fog status (host capacity + OS)** — agent `PUT /api/v3/agent/status` accepts and persists `systemCpus`, `systemTotalMemory`, `systemAvailableMemory`, `systemTotalDisk`, `systemAvailableDisk`, `systemOs`, `systemOsVersion`, and `systemKernelVersion`. User `GET /api/v3/iofog` and list return the stored values. `systemTotalCpu` remains host CPU busy **0–100%**; `diskUsage` remains Edgelet data-directory usage in **GiB**, not host filesystem totals. Columns on 3.9.0 baseline and 3.8→3.9 upgrade (sqlite, mysql, postgres). Omitted keys leave prior values unchanged. + ### Changed -- Embedded **EdgeOps Console** default version **v1.1.0-rc.4** → **v1.1.0-rc.5** +- Embedded **EdgeOps Console** default version **v1.1.0-rc.4** → **v1.1.0-rc.6** ## [v3.9.0-rc.7] From d149955867069a5b148c9e7955320d522f6ea152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 23 Sep 2026 15:41:18 +0300 Subject: [PATCH 54/55] Bump embedded EdgeOps Console default to v1.1.0-rc.6. Update Docker, Makefile, CI build env, dev console script, and .env.example to match the pinned console release. --- .env.example | 2 +- .github/actions/set-build-env/action.yml | 2 +- Dockerfile | 4 ++-- Makefile | 4 ++-- scripts/build-console-dev.js | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 70dcde88..1a775c40 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 +EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index c6d045dd..b022057a 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.5"; fi + if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.6"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/Dockerfile b/Dockerfile index 9a1591bf..7c8c0a65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -50,7 +50,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:e3392c129520d2ebce748c3d0048343d0ec60be560cf220b60a7ba5d2bd2484a -ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index 6ffabfaf..91905bef 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.5 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.5 +EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.6 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index 2f0dac8a..82c41bf6 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.5' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.6' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { From b7b38245c46bcddf6a122315b9cdebb3274ac936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Wed, 23 Sep 2026 15:43:22 +0300 Subject: [PATCH 55/55] changelog update --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffaa0d1f..107bf8b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,18 @@ # Changelog -## [v3.9.0-rc.8] +## [v3.9.0-rc.9] ### Added - **Fog status (host capacity + OS)** — agent `PUT /api/v3/agent/status` accepts and persists `systemCpus`, `systemTotalMemory`, `systemAvailableMemory`, `systemTotalDisk`, `systemAvailableDisk`, `systemOs`, `systemOsVersion`, and `systemKernelVersion`. User `GET /api/v3/iofog` and list return the stored values. `systemTotalCpu` remains host CPU busy **0–100%**; `diskUsage` remains Edgelet data-directory usage in **GiB**, not host filesystem totals. Columns on 3.9.0 baseline and 3.8→3.9 upgrade (sqlite, mysql, postgres). Omitted keys leave prior values unchanged. ### Changed -- Embedded **EdgeOps Console** default version **v1.1.0-rc.4** → **v1.1.0-rc.6** +- Embedded **EdgeOps Console** default version **v1.1.0-rc.5** → **v1.1.0-rc.6** + +## [v3.9.0-rc.8] + +### Changed +- Embedded **EdgeOps Console** default version **v1.1.0-rc.4** → **v1.1.0-rc.5** ## [v3.9.0-rc.7]