Switch BVT runs to petstore container (rather than permanent live web service) (#78)

Co-authored-by: stas <statis@microsoft.com>

Author: stishkin

Scripts/Tests/bvt-petstore.py renamed to Scripts/Tests/bvt-petstore3.py

@@ -33,30 +33,30 @@ def webhook_triggers_results(job_id, test_url):
 def time_span(t_start, t_end):
     return time.strftime("%H:%M:%S", time.gmtime(t_end - t_start))
 
-def bvt(cli, definitions):
+def bvt(cli, definitions, subs):
     print('Getting available wehook events')
     webhook_events = cli.list_available_webhooks_events()
     try:
         test_url = webhooks_test_url(definitions.subscription, definitions.resource_group, definitions.test_infra)
         for event in webhook_events:
             print(f'Setting webhook for {event}')
-            compile_webhook = cli.set_webhooks_subscription('petstore-compile', event, test_url)
-            fuzz_webhook = cli.set_webhooks_subscription('petstore-fuzz', event, test_url)
+            compile_webhook = cli.set_webhooks_subscription('petstore3-compile', event, test_url)
+            fuzz_webhook = cli.set_webhooks_subscription('petstore3-fuzz', event, test_url)
 
-        added_compile = cli.list_webhooks('petstore-compile', event)
+        added_compile = cli.list_webhooks('petstore3-compile', event)
         if len(added_compile) == 0:
-            raise Exception('Expected petstore-compile webhooks not to be empty after creation')
+            raise Exception('Expected petstore3-compile webhooks not to be empty after creation')
 
-        added_fuzz = cli.list_webhooks('petstore-fuzz', event)
+        added_fuzz = cli.list_webhooks('petstore3-fuzz', event)
         if len(added_fuzz) == 0:
-            raise Exception('Expected petstore-fuzz webhooks not to be empty after creation')
+            raise Exception('Expected petstore3-fuzz webhooks not to be empty after creation')
 
         t_pre_compile = time.time()
 
         print('Compile')
-        compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.compile.json'))
+        compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'compile.json'))
 
-        compile_config = raft.RaftJobConfig(file_path=compile_config_path)
+        compile_config = raft.RaftJobConfig(file_path=compile_config_path, substitutions=subs)
         compile_job = cli.new_job(compile_config)
         cli.poll(compile_job['jobId'], 10)
 
@@ -81,11 +81,9 @@ def bvt(cli, definitions):
                             f' {after_compile_pre_fuzz}')
 
         print('Fuzz')
-        fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.fuzz.json'))
-        subs = {}
+        fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'fuzz.json'))
         subs['{compile.jobId}'] = compile_job['jobId']
         fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs)
-        fuzz_config.config['duration'] = '00:20:00'
         fuzz_job = cli.new_job(fuzz_config)
         cli.poll(fuzz_job['jobId'], 10)
 
@@ -104,8 +102,8 @@ def bvt(cli, definitions):
                     raise Exception('Expected job to be in completed state when retrieved job list.'
                                     f'{after_fuzz}')
 
-        if m != 2:
-            raise Exception('Expected 2 after compile job step'
+        if m != 3:
+            raise Exception('Expected 3 after compile job step'
                             f' for job {fuzz_job["jobId"]}'
                             f' got {m}'
                             f' {after_fuzz}')
@@ -136,7 +134,7 @@ def bvt(cli, definitions):
         print('Validating that bugs posted events matches total bugs found in job status')
         total_bugs_found = 0
         for r in job_status_events:
-            if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId']:
+            if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId'] and r['Data']['Tool'] == 'RESTler':
                 total_bugs_found += r['Data']['Metrics']['TotalBugBucketsCount']
 
         print(f'Total bugs found: {total_bugs_found}')
@@ -161,12 +159,11 @@ def bvt(cli, definitions):
         if len(deleted_fuzz) > 0:
             raise Exception('Expected petstore-fuzz webhooks to be empty after deletion, instead got %s', deleted_compile)
 
-
-
 if __name__ == "__main__":
     formatter = argparse.ArgumentDefaultsHelpFormatter
     parser = argparse.ArgumentParser(description='bvt', formatter_class=formatter)
     raft.add_defaults_and_secret_args(parser)
+    parser.add_argument('--build', required=True)
     args = parser.parse_args()
 
     if args.defaults_context_json:
@@ -179,4 +176,10 @@ def bvt(cli, definitions):
     definitions = RaftDefinitions(defaults)
     defaults['secret'] = args.secret
     cli = RaftCLI(defaults)
-    bvt(cli, definitions)
+    subs = {
+        "{build-url}" : os.environ.get('SYSTEM_COLLECTIONURI'),
+        "{build-id}" : os.environ.get('BUILD_BUILDID'),
+        "{ci-run}" : args.build.replace('.', '-')
+    }
+    print(f"SUBS: {subs}")
+    bvt(cli, definitions, subs)

ado/stages/test/steps/bvt.yml

@@ -13,4 +13,4 @@ steps:
       azureSubscription: $(raft-subscription)
       scriptType: 'pscore'
       scriptLocation: 'inlineScript'
-      inlineScript: "python Scripts/Tests/bvt.py --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret) --host $(bvt-host)"
+      inlineScript: "python Scripts/Tests/bvt-petstore3.py --build $(Build.BuildNumber) --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret)"

cli/samples/restler/self-contained/swagger-petstore/restler.compile.json

@@ -3,10 +3,7 @@
     "URL" : "http://localhost:8080/api/swagger.json"
   },
   "webhook": {
-    "name": "petstore-compile",
-    "metadata": {
-	    "action" : "compile"
-    }
+    "name": "petstore-compile"
   },
   "testTargets" : {
     "targets" : [

cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json

@@ -1,12 +1,7 @@
 {
   "host": "localhost",
   "webhook": {
-    "name": "petstore-fuzz",
-    "metadata": {
-      "app" : "petstore",
-      "swagger_version" : "v2",
-      "action" : "fuzz"	
-    }
+    "name": "petstore-fuzz"
   },
 
   "readonlyFileShareMounts": [

cli/samples/restler/self-contained/swagger-petstore/restler.test.json

@@ -2,12 +2,7 @@
   "host": "localhost",
 
   "webhook": {
-    "name": "petstore",
-    "metadata": {
-      "app" : "petstore",
-      "swagger_version" : "v2",
-      "action" : "test"	
-    }
+    "name": "petstore-test"
   },
   "readonlyFileShareMounts": [
     {

cli/samples/restler/self-contained/swagger-petstore/run.py

@@ -14,8 +14,6 @@ def run(compile, test, fuzz, replay):
     cli = RaftCLI()
     # Create compilation job configuration
     compile_job_config = RaftJobConfig(file_path=compile)
-    # add webhook metadata that will be included in every triggered webhook by Compile job
-    compile_job_config.add_metadata({"branch":"wizbangFeature"})
     print('Compile')
     # submit a new job with the Compile config and get new job ID
     compile_job = cli.new_job(compile_job_config)
@@ -37,8 +35,6 @@ def run(compile, test, fuzz, replay):
     # create a new job config with Fuzz configuration JSON
     fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
     print('Fuzz')
-    # add webhook metadata that will included in every triggered webhook by Fuzz job
-    fuzz_job_config.add_metadata({"branch":"wizbangFeature"})
     # create new fuzz job configuration
     fuzz_job = cli.new_job(fuzz_job_config)
 

cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json renamed to cli/samples/restler/self-contained/swagger-petstore3/compile.json

@@ -1,18 +1,11 @@
-{
+{ 
+  "rootFileShare" : "{ci-run}",
+  "namePrefix" : "petstore3-compile-",
   "swaggerLocation": {
     "URL" : "http://localhost:8082/api/v3/openapi.json"
   },
 
-  "resources": {
-    "Cores": 4,
-    "MemoryGBs": 8
-  },
-
   "testTargets" : {
-    "resources" : {
-      "Cores" : 2,
-      "MemoryGBs" : 4
-    },
     "targets" : [
       {
         "Container" : "swaggerapi/petstore3",

cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json renamed to cli/samples/restler/self-contained/swagger-petstore3/fuzz.json

@@ -1,15 +1,24 @@
 {
+  "rootFileShare" : "{ci-run}",
+  "namePrefix" : "petstore3-test-fuzz-lean-",
   "host": "localhost",
 
   "resources": {
     "Cores": 4,
     "MemoryGBs": 8
   },
 
+  "webhook":{
+    "name" : "petstore3-fuzz",
+    "metadata" : {
+      "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results"
+    }
+  },
+
   "readonlyFileShareMounts": [
     {
-      "FileShareName": "{compile.jobId}",
-      "MountPath": "/job-compile"
+      "FileShareName": "{ci-run}",
+      "MountPath": "/ci-run"
     }
   ],
 
@@ -26,46 +35,41 @@
         "Run" : {
           "Command" : "/bin/sh",
           "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"]
-        },
-        "Shell" : "/bin/sh",
-        "OutputFolder" : "restler_petstore3"
+        }
       },
       {
         "Container" : "swaggerapi/petstore3",
-        "Port" : 8080,
-        "ExpectedDurationUntilReady" : "00:00:30",
-        "PostRun" : {
-          "Command" : "/bin/sh", 
-          "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"],
-          "ExpectedRunDuration" : "00:00:10"
-        },
-        "Shell" : "/bin/sh",
-        "OutputFolder" : "zap_petstore3"
+        "Port" : 8082,
+        "ExpectedDurationUntilReady" : "00:02:00",
+        "Run" : {
+          "Command" : "/bin/sh",
+          "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8082 /swagger-petstore/server.war"]
+        }
       }
     ]
   },
   "tasks": [
     {
       "toolName": "RESTler",
       "outputFolder": "fuzz",
-      "duration": "00:30:00",
+      "duration": "00:10:00",
       "toolConfiguration": {
         "task": "Fuzz",
         "runConfiguration": {
           "targetEndpointConfiguration" : {
             "Port" : 8081 
           },
           "useSsl" : false,
-          "inputFolderPath": "/job-compile/compile"
+          "inputFolderPath": "/ci-run/{compile.jobId}/compile"
         }
       }
-    },
+    },  
     {
-      "toolName" : "ZAP",
-      "outputFolder" : "zap",
+      "toolName": "ZAP",
+      "outputFolder": "zap",
       "swaggerLocation": {
-        "URL" : "http://localhost:8080/api/v3/openapi.json"
+        "URL" : "http://localhost:8082/api/v3/openapi.json"
       }
     }
   ]
-}
+}

cli/samples/restler/self-contained/swagger-petstore3/run.py

@@ -4,47 +4,60 @@
 import pathlib
 import sys
 import os
+import json
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..'))
-from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError
+sys.path.append(os.path.join(cur_dir, '..'))
+from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError, RaftDefinitions
 
-def run(compile, test, fuzz):
-    # instantiate RAFT CLI
-    cli = RaftCLI()
 
+def run(cli, config, subs):
     # Create compilation job configuration
-    compile_job_config = RaftJobConfig(file_path=compile)
-    print('Compile')
+    job_config = RaftJobConfig(file_path=config, substitutions=subs)
+    print(f'Running {config}')
     # submit a new job with the Compile config and get new job ID
-    compile_job = cli.new_job(compile_job_config)
+    job = cli.new_job(job_config)
     # wait for a job with ID from compile_job to finish the run
-    cli.poll(compile_job['jobId'])
-
-    # use compile job as input for fuzz job
-    subs = {}
-    subs['{compile.jobId}'] = compile_job['jobId']
-
-    test_job_config = RaftJobConfig(file_path=test, substitutions=subs)
-    print('Test')
-    # create new fuzz job configuration
-    test_job = cli.new_job(test_job_config)
-    # wait for job ID from fuzz_job to finish the run
-    cli.poll(test_job['jobId'])
-
-    # create a new job config with Fuzz configuration JSON
-    fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
-    print('Fuzz')
-    # create new fuzz job configuration
-    fuzz_job = cli.new_job(fuzz_job_config)
-
-    # wait for job ID from fuzz_job to finish the run
-    cli.poll(fuzz_job['jobId'])
+    cli.poll(job['jobId'])
+    return job['jobId']
 
 if __name__ == "__main__":
     try:
-        run(os.path.join(cur_dir, "restler.compile.json"),
-            os.path.join(cur_dir, "restler.test.json"), 
-            os.path.join(cur_dir, "restler.zap.fuzz.json"))
+        defaults = None
+
+        if sys.argv[1] == '--build':
+            build_id = sys.argv[2].replace(".", "-")
+        print(f"BUILD ID : {build_id}")
+
+        with open(os.path.join(cur_dir, '..', 'defaults.json'), 'r') as defaults_json:
+            defaults = json.load(defaults_json)
+            if sys.argv[3] == '--secret':
+                defaults['secret'] = sys.argv[4]
+
+        # instantiate RAFT CLI
+        cli = RaftCLI(defaults)
+        defs = RaftDefinitions(defaults)
+
+        compile_job_id = None
+        subs = {
+            "{ci-run}" : f"{build_id}",
+            "{build-url}" : os.environ['SYSTEM_COLLECTIONURI'],
+            "{build-id}" : os.environ['BUILD_BUILDID'],
+            "{raft-subscription}": defs.subscription,
+            "{raft-resource-group}" : defs.resource_group,
+            "{raft-storage-account}" : defs.storage_account
+        }
+        for arg in sys.argv[1:]:
+            if arg == 'compile':
+                compile_job_id = run(cli, os.path.join(cur_dir, 'compile.json'), subs)
+                subs['{compile.jobId}'] = compile_job_id
+
+            if arg == 'test':
+                run(cli, os.path.join(cur_dir, "test.json"), subs), 
+
+            if arg == 'test-fuzz-lean':
+                run(cli, os.path.join(cur_dir, "test-fuzz-lean.json"), subs), 
+
     except RaftJobError as ex:
-        print(f'ERROR: {ex.message}')
+        print(f'ERROR: {ex.message}')
+        sys.exit(1)