Merge branch 'feat/cleanup-logging' into develop

Author: dem1fo

package-lock.json

@@ -18,7 +18,9 @@ import fs from 'node:fs';
 import { attemptToReadTOMLData } from './utils';
 import type { AppConfigData } from './Config';
 import Debug from 'debug';
-const log = Debug('CID:appConfig');
+const log = Debug('cid::engine::config::appConfig');
+
+// TODO: This type of configuration should probably be included as part of the electron application, not the backend engine.
 
 const APP_CONFIG_ENCODING: fs.EncodingOption = 'utf-8';
 
@@ -38,14 +40,14 @@ export const DEFAULT_APP_CONFIG: AppConfigData = {
 };
 
 export function loadAppConfig(configPath: string) {
-  log('Loading Application config from', configPath);
+  log(`[INFO] Loading Application config from ${configPath}`);
 
   // attempt to read the file
   const configData = attemptToReadTOMLData<AppConfigData>(configPath, APP_CONFIG_ENCODING);
 
   // if cannot be read we assume default application configuration
   if (!configData) {
-    log('Cannot find Application config file -- using the default');
+    log('[WARN] Cannot find Application config file, using the default configuration');
     return DEFAULT_APP_CONFIG;
   }
 
@@ -56,7 +58,7 @@ export function loadAppConfig(configPath: string) {
     typeof configData.window.width !== 'number' ||
     typeof configData.window.height !== 'number'
   ) {
-    log('Application config file is not valid -- using the default');
+    log('[ERROR] Application config file is not valid, using the default configuration');
     return DEFAULT_APP_CONFIG;
   }
 
@@ -65,7 +67,7 @@ export function loadAppConfig(configPath: string) {
 
 export function saveAppConfig(configData: AppConfigData, outputPath: string) {
   // update the config hash on import to account for the
-  const outputData = JSON.stringify(configData, null, '    ');
+  const outputData = JSON.stringify(configData, null, 4);
   fs.writeFileSync(outputPath, outputData, APP_CONFIG_ENCODING);
-  log('Written Application config data to ', outputPath);
+  log(`[INFO] Written Application config data to '${outputPath}'`);
 }

src/config/appConfig.ts

@@ -16,8 +16,9 @@
 
 import { dirname } from 'node:path';
 import fs from 'node:fs';
+
 import Debug from 'debug';
-const log = Debug('CID:ConfigStore');
+const log = Debug('cid::engine::config::store');
 
 import { loadConfig, CONFIG_FILE_ENCODING } from './loadConfig';
 import { loadAppConfig, saveAppConfig, DEFAULT_APP_CONFIG } from './appConfig';
@@ -27,7 +28,7 @@ import type { AppConfigData, Config } from './Config';
 // Ensure the application's config file directory exists
 function ensureAppDirectoryExists(appDir: string) {
   if (!fs.existsSync(appDir)) {
-    log("Application directory '", appDir, "' does not exits -- creating it");
+    log(`[WARN] Application directory '${appDir}' does not exits, creating it`);
     fs.mkdirSync(appDir /*, { recursive: true } */);
   }
 }
@@ -40,7 +41,7 @@ function saveConfig(configData: Config.FileConfiguration, outputPath: string) {
   // update the config hash on import to account for the
   const outputData = JSON.stringify(configData, null, '    ');
   fs.writeFileSync(outputPath, outputData, CONFIG_FILE_ENCODING);
-  log('Written config data to ', outputPath);
+  log(`[INFO] Written config data to ${outputPath}`);
 }
 
 interface ConfigStorePaths {
@@ -96,12 +97,12 @@ export class ConfigStore {
     // if the load succesds we have a valid config -- use it as a
     // user-provided one
     if (userConfigLoad.success) {
-      log('User config validation success - using it as config');
+      log('[INFO] User config validation success - using it as config');
       this.useUserConfig(userConfigLoad.config, userConfigLoad.lastUpdated);
       return;
     }
 
-    log('User config validation not successful - attempting to load backup config');
+    log('[WARN] User config validation unsuccessful - attempting to load backup config');
     // if the default config load failed use the backup default
     // from the app distribution
     const backupConfigLoad = loadConfig({
@@ -114,7 +115,7 @@ export class ConfigStore {
     // if the load succesds we have a valid config -- use it as
     // a config-from-backup
     if (backupConfigLoad.success) {
-      log('Backup config validation success - using it as config');
+      log('[INFO] Backup config validation success - using it as config');
       backupConfigLoad.config.isBackup = true;
       this.useBackupConfig(backupConfigLoad.config);
       return;
@@ -124,10 +125,7 @@ export class ConfigStore {
     this.loadError = backupConfigLoad.error;
 
     // if the backup config fails to load we are screwed
-    log(
-      'Backup config load failed - the application should alert the user: ',
-      backupConfigLoad.error,
-    );
+    log(`[ERROR] Backup config load failed - the application should alert the user: ${backupConfigLoad.error}`);
 
     // if there is a salt file error store the config, but act like it's invalid
     // (this is needed to pick up error messages from the config file)
@@ -175,16 +173,16 @@ export class ConfigStore {
   // This method does not save the backup as the user config, only deletes the user config file
   removeUserConfig() {
     // attempt to load the backup config
-    log('[removeUserConfig] Attempting to remove user configuration and replace with backup.');
+    log('[INFO] Attempting to remove user configuration and replace with backup.');
 
     // if the current config is already a backup config don't do anything
     if (this.isCurrentConfigBackup()) {
-      log('[removeUserConfig] Already using a backup config -- bailing');
+      log('[INFO] Already using a backup config -- bailing');
       // this is not an error - we're already using the backup
       return;
     }
 
-    log('[removeUserConfig] Trying to load backup config file');
+    log('[INFO] Trying to load backup config file');
     const backupConfigLoad = loadConfig({
       configPath: this.getBackupConfigFilePath(),
       algorithmId: this.getAlgorithmId(),
@@ -194,18 +192,15 @@ export class ConfigStore {
 
     // if failed return the error message (do not delete the user config yet)
     if (!backupConfigLoad.success) {
-      log(
-        'Backup config validation failed -- returning error and keeping existing user config:',
-        backupConfigLoad.error,
-      );
+      log(`[ERROR] Backup config validation failed -- returning error and keeping existing user config: ${backupConfigLoad.error}`)
       // save the error
       this.loadError = backupConfigLoad.error;
       // and return it
       return backupConfigLoad.error;
     }
 
     // if successful use the loaded backup configuration
-    log('[removeUserConfig] Backup config validation success - using it as config');
+    log('[INFO] Backup config validation success - using it as config');
     backupConfigLoad.config.isBackup = true;
     this.useBackupConfig(backupConfigLoad.config);
 
@@ -246,7 +241,7 @@ export class ConfigStore {
   // deletes the user configuration file
   _deleteUserConfigFile() {
     const configPath = this.getConfigFilePath();
-    log('Deleting config file: ', configPath);
+    log(`[INFO] Deleting config file: ${configPath}`);
     fs.unlinkSync(configPath);
   }
 

src/config/configStore.ts

@@ -22,8 +22,9 @@ import { generateConfigHash } from './utils';
 
 import { attemptToReadTOMLData } from './utils';
 import type { Config } from './Config';
+
 import Debug from 'debug';
-const log = Debug('CID:loadConfig');
+const log = Debug('cid::engine::config::loadConfig');
 
 // The encoding used by the config file
 export const CONFIG_FILE_ENCODING: fs.EncodingOption = 'utf-8';
@@ -43,14 +44,14 @@ type LoadConfigInput = {
 }
 
 export function loadConfig({ configPath, algorithmId, embeddedSalt, usingUI=false, validateConfig=true }: LoadConfigInput): LoadConfigResult {
-  log('[INFO] Loading config from', configPath);
+  log(`[INFO] Loading config from '${configPath}'`);
   const configData = attemptToReadTOMLData<Config.FileConfiguration>(configPath, CONFIG_FILE_ENCODING);
 
   if (!configData) {
-    log('[ERROR] Unable to read config file', configPath);
+    log(`[ERROR] Unable to read config file from '${configPath}'`);
     return {
       success: false,
-      error: `Unable to read config file '${configPath}'`,
+      error: `Unable to read config file from '${configPath}'`,
       isSaltFileError: false
     };
   }
@@ -59,6 +60,7 @@ export function loadConfig({ configPath, algorithmId, embeddedSalt, usingUI=fals
   const lastUpdateDate = new Date(fs.statSync(configPath).mtime);
 
   if (!!validateConfig) {
+    log('[INFO] Validating config');
     // validate the config
     const validationResult = validateConfigFile(configData, algorithmId, usingUI);
 
@@ -76,6 +78,7 @@ export function loadConfig({ configPath, algorithmId, embeddedSalt, usingUI=fals
 
     // fail if the signature is not OK
     if (configHash !== configData.meta.signature) {
+      log(`[ERROR] Configuration file signature mismatch; config has '${configData.meta.signature}', generated '${configHash}'`);
       return {
         success: false,
         error: `Configuration file signature mismatch -- required signature is '${configData.meta.signature}' but user configuration has '${configHash}' `,
@@ -86,6 +89,7 @@ export function loadConfig({ configPath, algorithmId, embeddedSalt, usingUI=fals
 
   // alphabetically sort the process, static, and reference fields to standardise and prevent
   // different ordering in config producing different results in output.
+  log('[DEBUG] Standardising column order in configuration');
   configData.algorithm.columns.process = configData.algorithm.columns.process.sort();
   configData.algorithm.columns.reference = configData.algorithm.columns.reference.sort();
   configData.algorithm.columns.static = configData.algorithm.columns.static.sort();
@@ -95,22 +99,26 @@ export function loadConfig({ configPath, algorithmId, embeddedSalt, usingUI=fals
   // the config fields are optional. The programme should fail if no salt is provided.
 
   if (configData.algorithm.salt && configData.algorithm.salt.source == "STRING") {
+    log('[DEBUG] Using string-based salt from configuration file');
     return { success: true, lastUpdated: lastUpdateDate, config: configData };
   }
 
   if (configData.algorithm.salt && configData.algorithm.salt.source == "FILE") {
+    log('[DEBUG] Using file-based salt from configuration file');
     // load the file, convert to a string value, update the config to be of type: "STRING"
     const saltFilePath = configData.algorithm.salt.value;
     const validatorRegexp = configData.algorithm.salt.validator_regex ? new RegExp(configData.algorithm.salt.validator_regex) : undefined;
     return tryLoadSaltFile({ saltFilePath, validatorRegexp, configData, lastUpdateDate, label: "salt" });
   }
 
   if (embeddedSalt && embeddedSalt.source == "STRING") {
+    log('[DEBUG] Using string-based salt from embedded configuration');
     configData.algorithm.salt = { source: "STRING", value: embeddedSalt.value }
     return { success: true, lastUpdated: lastUpdateDate, config: configData };
   }
-
+  
   if (embeddedSalt && embeddedSalt.source == "FILE") {
+    log('[DEBUG] Using file-based salt from embedded configuration');
     const saltFilePath = embeddedSalt.value;
     const validatorRegexp = embeddedSalt.validator_regex ? new RegExp(embeddedSalt.validator_regex) : undefined;
     return tryLoadSaltFile({ saltFilePath, validatorRegexp, configData, lastUpdateDate, label: "embedded salt" });
@@ -128,7 +136,7 @@ interface TryLoadSaltFileInput {
 }
 
 function tryLoadSaltFile({ saltFilePath, validatorRegexp, configData, lastUpdateDate }: TryLoadSaltFileInput): LoadConfigResult {
-  log('[INFO] Loading salt from', saltFilePath);
+  log(`[INFO] Loading salt from ${saltFilePath}`);
 
   const loadSaltResponse = loadSaltFile({ saltFilePath, validatorRegexp });
   if (!loadSaltResponse.success) {
@@ -140,8 +148,11 @@ function tryLoadSaltFile({ saltFilePath, validatorRegexp, configData, lastUpdate
   if (loadSaltResponse.message) log(loadSaltResponse.message);
 
   // update the config to be of salt type: "STRING" with loaded file data
+  log(`[DEBUG] Updating configuration with string-based salt of ${loadSaltResponse.data.length} characters`);
   configData.algorithm.salt = { source: "STRING", value: loadSaltResponse.data }
+
   // update the signature since we have changed the salt configuration
   configData.meta.signature = generateConfigHash(configData);
+  log(`[DEBUG] Updated configuration signature to ${configData.meta.signature}`);
   return { success: true, lastUpdated: lastUpdateDate, config: configData };
 }

src/config/loadConfig.ts

@@ -14,8 +14,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import fs from 'node:fs';
-import path from 'node:path';
-
 import { attemptToReadFileData } from './utils';
 
 // the encoding used for the salt file

src/config/loadSaltFile.ts

@@ -14,9 +14,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import Debug from 'debug';
-const log = Debug('CID:Decoder');
-
 import type { Config } from '../config/Config';
 import type { CidDocument } from '../document';
 import type { RawData, MappedData } from '../document';
@@ -59,7 +56,6 @@ export abstract class DecoderBase {
       return transformed;
     });
 
-    log('AFTER:', objectRows[0]);
     return objectRows;
   }
 

src/decoding/base.ts

@@ -18,10 +18,12 @@ import fs from 'node:fs';
 import { parse as csv_parse } from 'csv-parse/sync';
 import type { Options as CsvOptions } from 'csv-parse/sync';
 
+import Debug from 'debug';
+const log = Debug('cid::engine::decoding::csv');
+
 import { DecoderBase } from './base';
 import type { Config } from '../config/Config';
 
-// A decoder for CSVs
 class CsvDecoder extends DecoderBase {
   csvOptions: CsvOptions = {};
 
@@ -31,9 +33,16 @@ class CsvDecoder extends DecoderBase {
   }
 
   decodeFile(path: string, fileEncoding: fs.EncodingOption = 'utf-8') {
-    let data = fs.readFileSync(path, fileEncoding);
-    let parsed = csv_parse(data, this.csvOptions);
-    return this.documentFromRawData(path, parsed);
+    log(`[INFO] Reading CSV file from ${path} with encoding '${fileEncoding}'`);
+    const data = fs.readFileSync(path, fileEncoding);
+    const parsed = csv_parse(data, this.csvOptions);
+    log(`[INFO] Parsed ${parsed.length} rows from CSV file '${path}'`);
+
+    log(`[DEBUG] Found ${parsed[0].length} columns: [${parsed[0].join(', ')}]`);
+    const document = this.documentFromRawData(path, parsed);
+    log(`[DEBUG] Renamed columns to aliases, columns: [${Object.keys(document.data[0]).join(', ')}]`);
+
+    return document;
   }
 }
 

src/decoding/csv.ts

@@ -27,7 +27,7 @@ export function fileTypeOf(filePath: string) {
   if (filePath.endsWith('.csv')) {
     return SUPPORTED_FILE_TYPES.CSV;
   }
-  throw new Error('Unknown file type');
+  throw new Error(`Unknown file type for file at '${filePath}'`);
 }
 
 // Returns an appropriate decoder for a file