Skip to content
This repository was archived by the owner on Feb 2, 2022. It is now read-only.

Commit 585a5dc

Browse files
stishkinstas
stishkin
and
stas
authored
Move TestAllCombinations into a mode (#221)
Co-authored-by: stas <statis@microsoft.com>
1 parent 558c924 commit 585a5dc

File tree

5 files changed

+39
-28
lines changed

5 files changed

+39
-28
lines changed

‎cli/raft-tools/tools/RESTler/schema.json‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -274,10 +274,6 @@
274274
"type": "boolean",
275275
"description": "Ignore feedback from response"
276276
},
277-
"testAllCombinations":{
278-
"type": "boolean",
279-
"description": "By default, test mode will try to execute each request successfully once. This means that, if there are 5 possible values for a parameter, and the request\r\nis successfully executed when passing the first value, the remaining 4 will not be tested.\r\nIn some cases, such as for differential regression testing, it is desired to test all of the specified parameter values in Test mode.\r\nSetting TestAllCombinations to true in test mode in order to try all parameter values\r\n(up to max_combinations).\r\nResults for all parameter combinations will be reported in the spec coverage file."
280-
},
281277
"includeUserAgent":{
282278
"type": "boolean",
283279
"description": "Include RESTler user agent"
@@ -339,7 +335,7 @@
339335
"type": "string",
340336
"description": "RESTLer task types",
341337
"nullable": false,
342-
"enum" : ["compile", "test", "fuzz", "replay", "testFuzzLean", "fuzzRandomWalk", "fuzzBfsCheap"]
338+
"enum" : ["compile", "test", "fuzz", "replay", "testFuzzLean", "testAllCombinations", "fuzzRandomWalk", "fuzzBfsCheap"]
343339
},
344340
"compileConfiguration": {
345341
"$ref": "#/components/schemas/CompileConfiguration"

‎cli/samples/restler/running-against-raft/test.json‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"tasks": [
1313
{
1414
"toolName": "RESTler",
15-
"outputFolder": "RESTLer-test-fuzz-lean-1",
15+
"outputFolder": "RESTler-test-fuzz-lean-1",
1616
"keyVaultSecrets": [ "RaftServicePrincipal" ],
1717
"authenticationMethod": {
1818
"MSAL": "RaftServicePrincipal"
@@ -28,7 +28,23 @@
2828
},
2929
{
3030
"toolName": "RESTler",
31-
"outputFolder": "RESTler-test-2",
31+
"outputFolder": "RESTler-test-all-combinations-2",
32+
"keyVaultSecrets": [ "RaftServicePrincipal" ],
33+
"authenticationMethod": {
34+
"MSAL": "RaftServicePrincipal"
35+
},
36+
"toolConfiguration": {
37+
"task": "TestAllCombinations",
38+
"runConfiguration": {
39+
"inputFolderPath": "/job-compile/RESTler-compile",
40+
"authenticationTokenRefreshIntervalSeconds": 300,
41+
"maxRequestExecutionTime" : 300
42+
}
43+
}
44+
},
45+
{
46+
"toolName": "RESTler",
47+
"outputFolder": "RESTler-test-3",
3248
"keyVaultSecrets": [ "RaftServicePrincipal" ],
3349
"authenticationMethod": {
3450
"MSAL": "RaftServicePrincipal"
@@ -37,8 +53,7 @@
3753
"task": "Test",
3854
"runConfiguration": {
3955
"inputFolderPath": "/job-compile/RESTler-compile",
40-
"maxRequestExecutionTime" : 300,
41-
"testAllCombinations" : true
56+
"maxRequestExecutionTime" : 300
4257
}
4358
}
4459
}

‎docs/sdk/swagger.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -895,7 +895,7 @@ see: https://docs.microsoft.com/en-us/azure/container-instances/container-instan
895895
</div>
896896
<div class="model">
897897
<h3><a name="RunConfiguration"><code>RunConfiguration</code></a> <a class="up" href="#__Models">Up</a></h3>
898-
<div class='model-description'>RESTler job Test, Fuzz or Replay configuration</div>
898+
<div class='model-description'>RESTler job Test, TestFuzzLean, TestAllCombinations, Fuzz or Replay configuration</div>
899899
<div class="field-items">
900900
<div class="param">grammarPy (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> Path to grammar py relative to compile folder path. If not set then default "grammar.py" grammar is assumed </div>
901901
<div class="param">inputFolderPath (optional)</div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> For Test or Fuzz tasks: Grammar is produced by compile step. The compile step
@@ -911,7 +911,6 @@ For Replay task: path to RESTler Fuzz or Test run that contains bug buckets to r
911911
<div class="param">maxRequestExecutionTime (optional)</div><div class="param-desc"><span class="param-type"><a href="#integer">Integer</a></span> Maximum request execution time </div>
912912
<div class="param">ignoreDependencies (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Ignore resource dependencies </div>
913913
<div class="param">ignoreFeedback (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Ignore feedback from responses </div>
914-
<div class="param">testAllCombinations (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> By default, test mode will try to execute each request successfully once. This means that, if there are 5 possible values for a parameter, and the request is successfully executed when passing the first value, the remaining 4 will not be tested. In some cases, such as for differential regression testing, it is desired to test all of the specified parameter values in Test mode. Setting TestAllCombinations to true in test mode in order to try all parameter values (up to max_combinations). Results for all parameter combinations will be reported in the spec coverage file </div>
915914
</div> <!-- field-items -->
916915
</div>
917916
<div class="model">

‎src/Agent/RESTlerAgent/AgentMain.fs‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -880,31 +880,35 @@ let main argv =
880880
match jobConfiguration.InputFolderPath with
881881
| Some p -> copyDir p workDirectory (set [taskConfigurationPath])
882882
| None -> ()
883-
884-
let testMode =
885-
match jobConfiguration.TestAllCombinations with
886-
| None | Some false -> "directed-smoke-test"
887-
| Some true -> "test-all-combinations"
888-
883+
let testMode = "directed-smoke-test"
884+
printfn "Running %s" testMode
885+
do! report Raft.JobEvents.Running (None, None)
886+
do! test testMode [] jobConfiguration
887+
let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
888+
return Raft.JobEvents.TaskCompleted, None, 0, summary
889+
890+
| TaskType.TestAllCombinations ->
891+
let jobConfiguration =
892+
match restlerPayload.RunConfiguration with
893+
| None -> RunConfiguration.Empty
894+
| Some jobConfiguration -> jobConfiguration
895+
match jobConfiguration.InputFolderPath with
896+
| Some p -> copyDir p workDirectory (set [taskConfigurationPath])
897+
| None -> ()
898+
let testMode = "test-all-combinations"
889899
printfn "Running %s" testMode
890900
do! report Raft.JobEvents.Running (None, None)
891901
do! test testMode [] jobConfiguration
892902
let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime
893903
return Raft.JobEvents.TaskCompleted, None, 0, summary
894-
895-
| TaskType.TestFuzzLean ->
896904

905+
| TaskType.TestFuzzLean ->
897906
printfn "Running test fuzz lean"
898907
let jobConfiguration =
899908
match restlerPayload.RunConfiguration with
900909
| None -> RunConfiguration.Empty
901910
| Some jobConfiguration -> jobConfiguration
902911

903-
match jobConfiguration.TestAllCombinations with
904-
| None | Some false ->
905-
printfn "WARNING: Test All Combination settings is not supported in test fuzz lean mode"
906-
| Some true -> ()
907-
908912
match jobConfiguration.InputFolderPath with
909913
| Some p -> copyDir p workDirectory (set [taskConfigurationPath])
910914
| None -> ()

‎src/Agent/RESTlerAgent/RESTlerAgentConfigTypes.fs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,6 @@ type RunConfiguration =
139139
/// Specifies whether to show contents of auth token in RESTler logs
140140
ShowAuthToken : bool option
141141

142-
/// Specifies whether to test all combinations in 'test' mode
143-
TestAllCombinations : bool option
144-
145142
/// Token Refresh Interval
146143
AuthenticationTokenRefreshIntervalSeconds: int option
147144

@@ -179,7 +176,6 @@ type RunConfiguration =
179176
GrammarPy = None
180177
InputFolderPath = None
181178
ProducerTimingDelay = None
182-
TestAllCombinations = None
183179
UseSsl = None
184180
AuthenticationTokenRefreshIntervalSeconds = None
185181
PathRegex = None
@@ -209,6 +205,7 @@ type TaskType =
209205
| Compile
210206
| Test
211207
| TestFuzzLean
208+
| TestAllCombinations
212209
| Fuzz
213210
| FuzzRandomWalk
214211
| FuzzBfsCheap

0 commit comments

Comments
 (0)