Update to Version v3.2.4 (#145)

Author: kamyarz-aws

CHANGELOG.md

@@ -5,16 +5,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [3.2.3] - 2023-10-5
+## [3.2.4] - 2023-11-10
+
+### Changed
+
+- Updating Lambda Runtimes to Use Node-18
+- Updated Version of Taurus image to 1.16.26
+- Updated Jmeter dependencies and taurus dependencis within the docker image to enhance the security of the docker image
+
+### Fixed
+
+- Bug fix to resolve issues with our recurring schedulers [#141](https://github.com/aws-solutions/distributed-load-testing-on-aws/issues/141)
+- Bug fix to resolve issues with deploying to distant regions as additional regions to the main region [#138](https://github.com/aws-solutions/distributed-load-testing-on-aws/issues/138)
+
+## [3.2.3] - 2023-10-05
 
 ### Fixed
- 
+
 - Bug fix to resolve Multipart upload for files bigger than 5MB
 - Bug fix to show success codes other than 200
-- Bug fix to reoslve [#131] https://github.com/aws-solutions/distributed-load-testing-on-aws/issues/131
+- Bug fix to reoslve [#131](https://github.com/aws-solutions/distributed-load-testing-on-aws/issues/131)
 - Updating CDK libraries to latest version to allow deployment in some form in Israel Region
 - Fixing Security Hotspots and resolving codesmells
-- Updating the Docker Image to the latest version 
+- Updating the Docker Image to the latest version
 
 ## [3.2.2] - 2023-06-29
 

CODE_OF_CONDUCT.md

@@ -1,5 +1,4 @@
 ## Code of Conduct
-
 This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
 For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
 opensource-codeofconduct@amazon.com with any additional questions or comments.

deployment/build-s3-dist.sh

@@ -143,13 +143,13 @@ echo "--------------------------------------------------------------------------
 echo "Creating container deployment package"
 echo "------------------------------------------------------------------------------"
 cd ${template_dir}/ecr/distributed-load-testing-on-aws-load-tester
-# Downloading jetty 9.4.34.v20201102
-curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-alpn-client/9.4.34.v20201102/jetty-alpn-client-9.4.34.v20201102.jar
-curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-alpn-openjdk8-client/9.4.34.v20201102/jetty-alpn-openjdk8-client-9.4.34.v20201102.jar
-curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-client/9.4.34.v20201102/jetty-client-9.4.34.v20201102.jar
-curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-http/9.4.34.v20201102/jetty-http-9.4.34.v20201102.jar
-curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-io/9.4.34.v20201102/jetty-io-9.4.34.v20201102.jar
-curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.34.v20201102/jetty-util-9.4.34.v20201102.jar
+# Downloading jetty 9.4.53.v20231009
+curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-alpn-client/9.4.53.v20231009/jetty-alpn-client-9.4.53.v20231009.jar
+curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-alpn-openjdk8-client/9.4.53.v20231009/jetty-alpn-openjdk8-client-9.4.53.v20231009.jar
+curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-client/9.4.53.v20231009/jetty-client-9.4.53.v20231009.jar
+curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-http/9.4.53.v20231009/jetty-http-9.4.53.v20231009.jar
+curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-io/9.4.53.v20231009/jetty-io-9.4.53.v20231009.jar
+curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.53.v20231009/jetty-util-9.4.53.v20231009.jar
 
 echo "------------------------------------------------------------------------------"
 echo "Building console"

deployment/ecr/distributed-load-testing-on-aws-load-tester/Dockerfile

@@ -3,11 +3,38 @@ FROM blazemeter/taurus:1.16.26
 RUN /usr/bin/python3 -m pip install --upgrade pip
 RUN pip install --no-cache-dir awscli
 RUN apt-get -y install xmlstarlet bc procps
+
+# Removing selenium and gatling from our image as they are not supported in DLT
+RUN rm -rf /root/.bzt/selenium-taurus
+RUN rm -rf /root/.bzt/gatling-taurus
+
+RUN apt-get update && apt-get upgrade -y
 # Taurus working directory  = /bzt-configs
 ADD ./load-test.sh /bzt-configs/
 ADD ./*.jar /bzt-configs/
 ADD ./*.py /bzt-configs/
+
 RUN chmod 755 /bzt-configs/load-test.sh
 RUN chmod 755 /bzt-configs/ecslistener.py
 RUN chmod 755 /bzt-configs/ecscontroller.py
+RUN chmod 755 /bzt-configs/jar_updater.py
+RUN python3 /bzt-configs/jar_updater.py
+
+# Fixing CVE-2023-22796
+WORKDIR /usr/local/rbenv/versions/3.2.2/lib/ruby/gems/3.2.0/gems/rbs-2.8.2/steep
+RUN sed -i 's/7.0.4/7.0.7.1/g' Gemfile.lock
+RUN gem install activesupport -v 7.0.7.1
+
+# Replacing urllib3, Werkzeug and cryptography with more stable Versions to fix vulnerabilities
+RUN pip install urllib3==2.0.7
+RUN pip install Werkzeug==3.0.1
+RUN pip install cryptography==41.0.5
+RUN rm -rf /root/.bzt/python-packages/3.10.12/urllib3*
+RUN rm -rf /root/.bzt/python-packages/3.10.12/werkzeug*
+RUN rm -rf /root/.bzt/python-packages/3.10.12/cryptography*
+RUN cp -r /usr/local/lib/python3.10/dist-packages/urllib3* /root/.bzt/python-packages/3.10.12/
+RUN cp -r /usr/local/lib/python3.10/dist-packages/werkzeug* /root/.bzt/python-packages/3.10.12/
+RUN cp -r /usr/local/lib/python3.10/dist-packages/cryptography* /root/.bzt/python-packages/3.10.12/
+
+WORKDIR /bzt-configs/
 ENTRYPOINT ["./load-test.sh"]

deployment/ecr/distributed-load-testing-on-aws-load-tester/jar_updater.py

@@ -0,0 +1,94 @@
+
+import os
+import time
+import requests
+
+from bzt.modules.jmeter import JarCleaner
+
+
+"""
+jar_updater.py updates json-smart, neo4j-java-driver, xalan to address CVEs on the taurus image. this is not DLT application code. 
+the script may be removed once taurus updates the libraries on the image.
+Affected Jmeter jars:
+    * json-smart v2.4.8 will be replaced with v2.4.9
+    * neo4j-java-driver v4.12.0 will be replaced with v5.14.0
+    * xalan v2.7.2 will be replaced with v2.7.3
+"""
+
+# these jars should be replaced with newer version in order to fix some vulnerabilities
+# component name and download link in https://repo1.maven.org/maven2/
+# These are Components with regards to JMETER
+JMETER_COMPONENTS = {
+    "json-smart": "net/minidev/json-smart/2.4.9/json-smart-2.4.9.jar",
+    "neo4j-java-driver": "org/neo4j/driver/neo4j-java-driver/5.14.0/neo4j-java-driver-5.14.0.jar",
+    "xalan": "xalan/xalan/2.7.3/xalan-2.7.3.jar",
+}
+JMETER_VERSION = "5.4.3"
+
+# To add other platform, and what to update, add affected components and version HERE
+
+def download(url, target_path):
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+    # Write the content to a file
+    with open(target_path, 'wb') as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+
+class Platform:
+    def __init__(self, platform, version, affected_components=None):
+        # To dynamically import from bzt.modules.gatling import Platform Objects
+        module = __import__('bzt.modules.' + platform.lower(), fromlist=[platform])
+        # Object created for the platform i.e JMeter(), Gatling()
+        self.obj = getattr(module, platform)()
+        self.lib_dir = f"/root/.bzt/{platform.lower()}-taurus/{version}/lib"
+        self.affected_components = affected_components
+
+    def install_jmeter_plugins(self):
+        plugins_mgr_version = "1.10"
+        plugins_mgr_link = f'https://search.maven.org/remotecontent?filepath=kg/apc/jmeter-plugins-manager/{plugins_mgr_version}/jmeter-plugins-manager-{plugins_mgr_version}.jar'
+        plugins_mgr_name = os.path.basename(plugins_mgr_link)
+        pm_installer_path = os.path.join(self.lib_dir, 'ext', plugins_mgr_name)
+        download(plugins_mgr_link, pm_installer_path)
+        self.obj._JMeter__install_plugins_manager(pm_installer_path)
+        cleaner = JarCleaner(self.obj.log)
+        cleaner.clean(os.path.join(self.lib_dir, 'ext'))
+
+    def _install_plugins(self):
+        dest = os.path.dirname(self.lib_dir)
+        plugins_manager_cmd = os.path.join(dest, 'bin', 'PluginsManagerCMD.sh')
+        cmd_line = [plugins_manager_cmd, 'install', ",".join([])]
+
+        try:
+            out, err = self.obj.call(cmd_line)
+        except Exception as exc:
+            raise Exception(f"Failed to install plugins : {exc}") #NOSONAR
+
+        self.log.debug("Install plugins: %s / %s", out, err)
+
+        if out and "Plugins manager will apply some modifications" in out:
+            time.sleep(5) 
+
+    def update_jars(self):
+        jar_files = [_file for _file in os.listdir(self.lib_dir) if _file.endswith(".jar")]
+        for jar_file in jar_files:
+            for comp_name in self.affected_components:
+                if jar_file.startswith(comp_name):
+                    download_link = "https://repo1.maven.org/maven2/" + self.affected_components[comp_name]
+                    target_path = os.path.join(self.lib_dir, os.path.basename(download_link))
+                    download(download_link, target_path)
+                    break
+                
+        cleaner = JarCleaner(self.obj.log)
+        cleaner.clean(self.lib_dir)
+
+
+if __name__ == "__main__":
+    # update this map to add other platforms
+    platform_components_map = {"JMeter": (JMETER_VERSION, JMETER_COMPONENTS)}
+    for key, value in platform_components_map.items():
+        platform = Platform(key, value[0], value[1])
+        platform.update_jars()
+        if key == "JMeter":
+            platform.install_jmeter_plugins()
+    

source/.eslintrc

@@ -32,10 +32,7 @@
       "rules": {
         "@typescript-eslint/no-inferrable-types": ["off", { "ignoreParameters": true, "ignoreProperties": true }],
         "@typescript-eslint/no-useless-constructor": ["off"],
-        "@typescript-eslint/no-unused-vars": [
-          "error",
-          { "args": "none", "argsIgnorePattern": "^_", "varsIgnorePattern": "^_" }
-        ],
+        "@typescript-eslint/no-unused-vars": ["error", { "args": "none", "argsIgnorePattern": "^_", "varsIgnorePattern": "^_" }],
         "@typescript-eslint/no-throw-literal": ["error"],
         "no-new": 0
       }
@@ -54,17 +51,12 @@
     "no-useless-constructor": ["off"],
     "no-throw-literal": ["off"],
 
-    "header/header": [
-      "error",
-      "line",
-      [" Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.", " SPDX-License-Identifier: Apache-2.0"],
-      2
-    ],
+    "header/header": ["error", "line", [" Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.", " SPDX-License-Identifier: Apache-2.0"], 2],
 
     "jsdoc/require-param-type": ["off"],
     "jsdoc/require-returns-type": ["off"],
     "jsdoc/newline-after-description": ["off"],
 
     "import/no-unresolved": 1 // warn only on Unable to resolve path import/no-unresolved
   }
-}
+}

source/.prettierrc.yml

@@ -8,4 +8,4 @@ trailingComma: "es5"
 tabWidth: 2
 semi: true
 quoteProps: "as-needed"
-printWidth: 120
+printWidth: 120

source/api-services/index.js

@@ -104,7 +104,7 @@ exports.handler = async (event, context) => {
   console.log(JSON.stringify(event, null, 2));
   let data;
   let response;
-  const config = JSON.parse(event.body);
+  let config = JSON.parse(event.body);
   const apiHandler = new APIHandler(event.resource, event.httpMethod);
 
   try {
@@ -113,6 +113,8 @@ exports.handler = async (event, context) => {
         data = await apiHandler.handleRegions();
         break;
       case "/scenarios":
+        if (!event.headers) config.eventBridge = "true";
+
         data = await apiHandler.handleScenarios(
           config,
           event.queryStringParameters,

source/api-services/lib/scenarios/index.js

@@ -449,32 +449,32 @@ const setTestId = (testId) =>
   testId || utils.generateUniqueId(10);
 /**
  * Sets the next schedule test run
+ * @param {string} scheduledTime
  * @param {string} scheduleRecurrence
  * @returns next run, scheduleRecurrence
  */
-const setNextRun = (scheduleRecurrence = "") => {
+const setNextRun = (scheduledTime, scheduleRecurrence = "") => {
   let nextRun = "";
   if (scheduleRecurrence) {
     switch (scheduleRecurrence) {
       case "daily":
-        nextRun = moment().utc().add(1, "d").format("YYYY-MM-DD HH:mm:ss");
+        nextRun = scheduledTime.add(1, "d").format("YYYY-MM-DD HH:mm:ss");
         break;
       case "weekly":
-        nextRun = moment().utc().add(7, "d").format("YYYY-MM-DD HH:mm:ss");
+        nextRun = scheduledTime.add(7, "d").format("YYYY-MM-DD HH:mm:ss");
         break;
       case "biweekly":
-        nextRun = moment().utc().add(14, "d").format("YYYY-MM-DD HH:mm:ss");
+        nextRun = scheduledTime.add(14, "d").format("YYYY-MM-DD HH:mm:ss");
         break;
       case "monthly":
-        nextRun = moment().utc().add(1, "M").format("YYYY-MM-DD HH:mm:ss");
+        nextRun = scheduledTime.add(1, "M").format("YYYY-MM-DD HH:mm:ss");
         break;
       default:
         throw new ErrorException("InvalidParameter", "Invalid recurrence value.");
     }
   }
-  return { nextRun, scheduleRecurrence };
+  return nextRun;
 };
-
 /**
  * Validates the setting for task count and task concurrency
  * @param {object} testTaskConfigs
@@ -758,16 +758,28 @@ const updateTestDBEntry = async (updateTestConfigs) => {
  */
 const createTest = async (config) => {
   console.log(`Create test: ${JSON.stringify(config, null, 2)}`);
-
   try {
     const { testName, testDescription, testType, showLive, regionalTaskDetails } = config;
-    let { testId, testScenario, testTaskConfigs, fileType } = config;
-
+    let { testId, testScenario, testTaskConfigs, fileType, scheduleTime, eventBridge, recurrence } = config;
+    let nextRun;
     fileType = setFileType(testType, fileType);
     testId = setTestId(testId);
 
-    const startTime = moment().utc().format("YYYY-MM-DD HH:mm:ss");
-    let { nextRun, scheduleRecurrence } = setNextRun(config.recurrence);
+    const testEntry = await getTestEntry(testId);
+    if (testEntry && testEntry.nextRun) nextRun = moment.utc(testEntry.nextRun, "YYYY-MM-DD HH:mm:ss");
+
+    let startTime = moment().utc();
+    if (eventBridge) {
+      const startDate = moment().format("YYYY-MM-DD");
+      // If it is eventBridge triggered definitely has scheduleTime
+      startTime = moment.utc(`${startDate} ${scheduleTime}:00`, "YYYY-MM-DD HH:mm:ss");
+    }
+
+    if (nextRun && startTime.isBefore(nextRun)) nextRun = nextRun.format("YYYY-MM-DD HH:mm:ss");
+    else nextRun = setNextRun(startTime, recurrence);
+
+    const scheduleRecurrence = recurrence ? recurrence : "";
+    startTime = startTime.format("YYYY-MM-DD HH:mm:ss");
 
     testTaskConfigs = validateTaskCountConcurrency(testTaskConfigs, regionalTaskDetails);
 
@@ -996,7 +1008,7 @@ const getTestHistoryTestRunIds = async (testId) => {
     };
     do {
       const testRunIds = await dynamoDB.query(params).promise();
-      testRunIds.Items.map((testRunItem) => {
+      testRunIds.Items.forEach((testRunItem) => {
         response.push(testRunItem.testRunId);
       });
       params.ExclusiveStartKey = testRunIds.LastEvaluatedKey;