[OGUI-1628] Remove usage of individual gRPC streams with ECS and base calibraiton page on kafka

Control/lib/api.js

@@ -103,7 +103,7 @@ module.exports.setup = (http, ws) => {
   const wsService = new WebSocketService(ws);
   const broadcastService = new BroadcastService(ws);
   const cacheService = new CacheService(broadcastService);
-  const environmentCacheService = new EnvironmentCacheService(broadcastService, eventEmitter);
+  const environmentCacheService = new EnvironmentCacheService(broadcastService, eventEmitter, cacheService);
   const qcConfigurationService = new QCConfigurationService(consulService);
 
   const qcConfigurationController = new QCConfigurationController(qcConfigurationService, config.consul);
@@ -121,10 +121,10 @@ module.exports.setup = (http, ws) => {
 
   const detectorService = new DetectorService(ctrlProxy, apricotProxy);
   const environmentService = new EnvironmentService(
-    ctrlProxy, apricotService, cacheService, broadcastService, environmentCacheService
+    ctrlProxy, detectorService, cacheService, broadcastService, environmentCacheService
   );
   const workflowService = new WorkflowTemplateService(ctrlProxy, apricotService);
-  const deploymentService = new DeploymentService(environmentService, workflowService, environmentCacheService);
+  const deploymentService = new DeploymentService(environmentService, workflowService, environmentCacheService, cacheService, broadcastService);
   const taskService = new TaskService(ctrlProxy);
 
   /**
@@ -179,6 +179,13 @@ module.exports.setup = (http, ws) => {
   const verifyLockOwnershipMiddleware = getDetectorsLockOwnershipMiddlewareFactory(lockService);
   const validateConsulServiceMiddleware = validateConsulServiceMiddlewareFactory(consulService);
   const verifyDetectorsAvailabilityMiddleware = verifyDetectorsAvailabilityMiddlewareFactory(detectorService);
+  const deploymentMandatoryMiddleware = [
+    ...coreMiddleware,
+    logDeploymentRequestMiddleware,
+    minimumRoleMiddleware(Role.DETECTOR),
+    verifyLockOwnershipMiddleware,
+    verifyDetectorsAvailabilityMiddleware,
+  ];
 
   ctrlProxy.methods.forEach(
     (method) => http.post(`/${method}`, coreMiddleware, (req, res) => ctrlService.executeCommand(req, res)),
@@ -196,7 +203,7 @@ module.exports.setup = (http, ws) => {
 
   http.get('/environments', coreMiddleware, envCtrl.getEnvironmentsHandler.bind(envCtrl), {public: true});
   http.get('/environment/:id/:source?', coreMiddleware, envCtrl.getEnvironmentHandler.bind(envCtrl), {public: true});
-  http.post('/environment/auto', coreMiddleware, envCtrl.newAutoEnvironmentHandler.bind(envCtrl));
+
   http.put('/environment/:id', 
     coreMiddleware,
     minimumRoleMiddleware(Role.DETECTOR),
@@ -213,14 +220,8 @@ module.exports.setup = (http, ws) => {
     envCtrl.destroyEnvironmentHandler.bind(envCtrl),
   );
 
-  http.post('/deploy',
-    coreMiddleware,
-    logDeploymentRequestMiddleware,
-    minimumRoleMiddleware(Role.DETECTOR),
-    verifyLockOwnershipMiddleware,
-    verifyDetectorsAvailabilityMiddleware,
-    deploymentController.newAsyncDeploymentHandler.bind(deploymentController)
-  );
+  http.post('/deploy', deploymentMandatoryMiddleware, deploymentController.newAsyncDeploymentHandler.bind(deploymentController));
+  http.post('/deploy/calibration', deploymentMandatoryMiddleware, deploymentController.newAsyncDeploymentCalibrationHandler.bind(deploymentController));
 
   http.delete('/deploy/:id',
     minimumRoleMiddleware(Role.DETECTOR),

Control/lib/controllers/Deployment.controller.js

@@ -19,6 +19,7 @@ const {
 } = require('@aliceo2/web-ui');
 
 const {User} = require('./../dtos/User.js');
+const LOG_FACILITY = 'cog/deployment-ctrl';
 
 /**
  * Controller Class for managing deployments via the AliECS system
@@ -121,6 +122,58 @@ class DeploymentController {
     }
   }
 
+  /**
+   * API - POST endpoint for requesting a new deployment for calibration purposes. 
+   * This is a specific endpoint separated from the generic deployment one as it has specific input requirements and validations.
+   * @param {Request} req - HTTP Request object which expects a body with the following properties
+   * @param {string[]} req.body.detectors - list of detectors for which the calibration environment should be deployed. Must contain exactly one detector.
+   * @param {string} req.body.runType - the type of the calibration run to be performed which determines the workflow template to use
+   * @param {string} req.body.configurationName - the name of the saved configuration to use for the deployment
+   * @param {Response} res - HTTP Response object with result of the deployment request
+   * @returns {void}
+   */
+  async newAsyncDeploymentCalibrationHandler(req, res) {
+    const {personid, name, username} = req.session;
+    const user = new User(username, name, personid);
+    const { detectors, runType, selectedConfiguration } = req.body;
+
+    if (detectors?.length !== 1) {
+      updateAndSendExpressResponseFromNativeError(
+        res,
+        new InvalidInputError('Exactly one detector must be specified for deployment')
+      );
+      return;
+    }
+    const [detector] = detectors;
+
+    if (!selectedConfiguration) {
+      updateAndSendExpressResponseFromNativeError(
+        res,
+        new InvalidInputError('Missing Configuration Name for deployment')
+      );
+      return;
+    }
+
+    // Attempt to deploy environment
+    try {
+      this._logger.infoMessage(`Request by username(${username}) to deploy configuration ${selectedConfiguration}`,
+        {level: LogLevel.OPERATIONS, system: 'GUI', facility: LOG_FACILITY}
+      );
+      const environment = await this._deploymentService.deployEnvironmentCalibration(
+        {
+          detector, runType, selectedConfiguration, user,
+        }
+      );
+      res.status(201).json(environment);
+    } catch (error) {
+      this._logger.errorMessage(
+        `Unable to deploy request by username(${username}) for ${selectedConfiguration} due to ${error.message}`,
+        {level: LogLevel.OPERATIONS, system: 'GUI', facility: LOG_FACILITY}
+      );
+      updateAndSendExpressResponseFromNativeError(res, error);
+    }
+  }
+
   /**
    * API - DELETE endpoint for acknowledging an environment deployment failure
    * @param {Request} req - HTTP Request object which expects an `id` as mandatory parameter

Control/lib/controllers/Environment.controller.js

@@ -12,9 +12,7 @@
  * or submit itself to any jurisdiction.
 */
 const {LogManager, LogLevel} = require('@aliceo2/web-ui');
-const {
-  updateAndSendExpressResponseFromNativeError, InvalidInputError, UnauthorizedAccessError
-} = require('@aliceo2/web-ui');
+const {updateAndSendExpressResponseFromNativeError, InvalidInputError} = require('@aliceo2/web-ui');
 
 const LOG_FACILITY = 'cog/env-ctrl';
 const {EnvironmentTransitionType} = require('./../common/environmentTransitionType.enum.js');
@@ -165,87 +163,6 @@ class EnvironmentController {
       this._logger.debug(`DESTROY_ENVIRONMENT,${id},${runNumber},${destroyRequestedAt},${Date.now()}`);
     }
   }
-
-  /**
-   * API - POST endpoint for deploying a new environment based on a given configuration name
-   * @param {Request} req - HTTP Request object
-   * @param {Response} res - HTTP Response object with EnvironmentDetails
-   * @returns {void}
-   */
-  async newAutoEnvironmentHandler(req, res) {
-    const {personid, name, username} = req.session;
-    const user = new User(username, name, personid);
-    const {detector, runType, configurationName} = req.body;
-
-    if (!this._lockService.isLockOwnedByUser(detector, user)) {
-      updateAndSendExpressResponseFromNativeError(res, new UnauthorizedAccessError('Lock not taken'));
-      return;
-    }
-
-    if (!configurationName) {
-      updateAndSendExpressResponseFromNativeError(
-        res,
-        new InvalidInputError('Missing Configuration Name for deployment')
-      );
-      return;
-    }
-
-    try {
-      const areDetectorsAvailable = await this._detectorService.areDetectorsAvailable([detector]);
-      if (!areDetectorsAvailable) {
-        updateAndSendExpressResponseFromNativeError(
-          res,
-          new InvalidInputError(`Detector ${detector} is already active`)
-        );
-        return;
-      }
-    } catch (error) {
-      updateAndSendExpressResponseFromNativeError(res, error);
-      return;
-    }
-
-    // Retrieve latest configuration version for given name
-    let variables;
-    try {
-      const configuration = await this._workflowService.retrieveWorkflowSavedConfiguration(configurationName);
-      if (!configuration.variables) {
-        throw new InvalidInputError(`No configuration variables found for ${configurationName}`);
-      }
-      variables = configuration.variables;
-    } catch (error) {
-      this._logger.debug(`Unable to retrieve saved configuration for ${configurationName} due to`);
-      this._logger.debug(error);
-      updateAndSendExpressResponseFromNativeError(res, error);
-      return;
-    }
-
-    // Retrieve latest default workflow to use
-    let workflowTemplatePath;
-    try {
-      const {template, repository, revision} = await this._workflowService.getDefaultTemplateSource();
-      workflowTemplatePath = `${repository}/workflows/${template}@${revision}`;
-    } catch (error) {
-      this._logger.debug(`Unable to retrieve default workflow template due to ${error}`);
-      updateAndSendExpressResponseFromNativeError(res, error);
-      return;
-    }
-    // Attempt to deploy environment
-    try {
-      this._logger.infoMessage(`Request by username(${username}) to deploy configuration ${configurationName}`,
-        {level: LogLevel.OPERATIONS, system: 'GUI', facility: LOG_FACILITY}
-      );
-      const environment = await this._envService.newAutoEnvironment(
-        workflowTemplatePath, variables, detector, runType, user
-      );
-      res.status(200).json(environment);
-    } catch (error) {
-      this._logger.errorMessage(
-        `Unable to deploy request by username(${username}) for ${configurationName} due to error`,
-        {level: LogLevel.OPERATIONS, system: 'GUI', facility: LOG_FACILITY}
-      );
-      updateAndSendExpressResponseFromNativeError(res, error);
-    }
-  }
 }
 
 module.exports = {EnvironmentController};

Control/lib/services/Deployment.service.js

@@ -12,8 +12,9 @@
  * or submit itself to any jurisdiction.
 */
 
-const {LogManager, LogLevel, NotFoundError} = require('@aliceo2/web-ui');
+const {LogManager, LogLevel, NotFoundError, InvalidInputError} = require('@aliceo2/web-ui');
 const CoreUtils = require('./../control-core/CoreUtils.js');
+const {CacheKeys} = require('../common/cacheKeys.enum.js');
 
 /**
  * **high-level service for deployment**
@@ -29,11 +30,16 @@ class DeploymentService {
    * Constructor for inserting dependencies needed to retrieve environment data
    * @param {EnvironmentService} environmentService - to use for creating new environments
    * @param {WorkflowService} workflowService - to use for retrieving template workflow information
+   * @param {EnvironmentCacheService} environmentCacheService - to use for retrieving and updating environment data in cache
+   * @param {CacheService} cacheService - to use for retrieving and updating general data in cache, e.g. calibration runs requests
+   * @param {BroadcastService} broadcastService - to use for broadcasting updates to clients, e.g. calibration runs requests updates
    */
-  constructor(environmentService, workflowService, environmentCacheService) {
+  constructor(environmentService, workflowService, environmentCacheService, cacheService, _broadcastService) {
     this._environmentService = environmentService;
     this._workflowService = workflowService;
     this._environmentCacheService = environmentCacheService;
+    this._generalCacheService = cacheService;
+    this._broadcastService = _broadcastService;
 
     this._logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'cog'}/deployment-service`);
   }
@@ -66,6 +72,69 @@ class DeploymentService {
     return environment;
   }
 
+  /**
+   * High-level service method to gather the necessary information and request the deployment of a calibration environment for a given detector and run type.
+   * @param {object} deploymentConfiguration - the configuration to be used for deployment
+   * @param {string} deploymentConfiguration.detector - the detector for which the calibration environment should be deployed
+   * @param {string} deploymentConfiguration.runType - the run type to be used for deployment, needed to retrieve the hosts to ignore for deployment
+   * @param {string} deploymentConfiguration.selectedConfiguration - the name of the saved configuration to be used for deployment, needed to retrieve the variables for deployment
+   * @param {User} deploymentConfiguration.user - the user to be used for deployment
+   * @returns {EnvironmentInfo} - the id of the environment created
+   * @throws {Error} - if the deployment fails or invalid input
+   */
+  async deployEnvironmentCalibration({ detector, runType, selectedConfiguration, user }) {
+    // Retrieve latest configuration version for given name
+    const { variables } = await this._workflowService.retrieveWorkflowSavedConfiguration(selectedConfiguration);
+    if (!variables) {
+      throw new InvalidInputError(`No configuration variables found for ${selectedConfiguration}`);
+    }
+
+    // Retrieve latest default workflow to use
+    const { template, repository, revision } = await this._workflowService.getDefaultTemplateSource();
+    const workflowTemplatePath = `${repository}/workflows/${template}@${revision}`;
+
+    const environment = await this._environmentService.newEnvironmentAsync({
+      workflowTemplate: workflowTemplatePath,
+      userVars: variables,
+      user,
+      shouldAutoTransition: true,
+      detectors: [detector],
+    });
+
+    /**
+     * A dedicated calibration page environment object is created to follow only change of environment state events
+     */
+    const calibrationEnvironment = {
+      inProgress: true,
+      detector,
+      runType,
+      events: [
+        {
+          type: 'ENVIRONMENT',
+          payload: {
+            id: environment.id,
+            message: 'request was sent to AliECS',
+            at: Date.now(),
+          }
+        }
+      ],
+    };
+    let calibrationRunsRequests = this._generalCacheService.getByKey(CacheKeys.CALIBRATION_RUNS_REQUESTS);
+    if (!calibrationRunsRequests) {
+      calibrationRunsRequests = {};
+    }
+    if (!calibrationRunsRequests[detector]) {
+      calibrationRunsRequests[detector] = {};
+    }
+    if (!calibrationRunsRequests[detector][runType]) {
+      calibrationRunsRequests[detector][runType] = calibrationEnvironment;
+
+    }
+    this._generalCacheService.updateByKeyAndBroadcast(CacheKeys.CALIBRATION_RUNS_REQUESTS, calibrationRunsRequests);
+    this._broadcastService.broadcast(CacheKeys.CALIBRATION_RUNS_REQUESTS, calibrationRunsRequests[detector][runType]);
+    return environment;
+  }
+
   /**
    * Method to acknowledge a deployment failure for a given environment. 
    * A failed deployment is not considered active anymore by ECS, thus it will only be present in the GUI cache