Updated generateResults to handle rpki-commons, added updated version of RQ4

Author: ameka4

artifacts/experiments/RQ4/generateResults.py

@@ -32,7 +32,13 @@
     "CharClassesQuickcheck#addString": 'addString',
     "StateSetQuickcheck#addStateDoesNotRemove": 'add',
     "StateSetQuickcheck#containsElements": 'contains',
-    "StateSetQuickcheck#removeAdd": 'remove'
+    "StateSetQuickcheck#removeAdd": 'remove',
+    "RoaCMSBuilderPropertyTest#buildEncodedParseCheck": 'roa',
+    "ManifestCMSBuilderPropertyTest#buildEncodedParseCheck": 'manifest',
+    "AspaCmsTest#should_generate_aspa": 'aspa',
+    "X509ResourceCertificateParentChildValidatorTest#validParentChildSubResources": 'resources',
+    "X509ResourceCertificateParentChildValidatorTest#validParentChildOverClaiming": 'claiming',
+    "X509ResourceCertificateParentChildValidatorTest#validParentChildOverClaimingLooseValidation": 'loose'
 }
 
 row_count = 1
@@ -46,6 +52,8 @@ def filter_for_recent_results(project_name: str, stats_directories: list[str]) -
     project_string = project_name if project_name != "convex" else project_name + "-core"  # edge case
     if "mph-table-fixed" in stats_directories[0]:  # edge case
         project_string = "mph-table-fixed"
+    elif "rpki-commons-fixed" in stats_directories[0]:
+        project_string = "rpki-commons-fixed"
     time_stamps = [datetime.datetime.strptime(x.replace(project_string, "").replace("_", ":").replace("T", " "), "%Y-%m-%d %H:%M:%S.%f")
                    for x in stats_directories]
     time_stamps.sort()
@@ -89,11 +97,14 @@ def generate_report_stats(stat_values: dict[str, dict]) -> dict[str, str]:
     property_dict = {}
     for key in first_iteration:
         property_dict[key] = []
-
+    
     # populate the dictionary with our results
     for key, val in stat_values.items():
         for prop, time in val.items():
             property_array = property_dict.get(prop)
+            if property_array is None:
+                property_dict[prop] = []
+                property_array = property_dict.get(prop)
             property_array.append(time)
 
     # generate mean, standard deviation and populate our final object
@@ -142,7 +153,7 @@ def main():
         fixed_stats_directories = obtain_stats_directories(results_directory=fixed_results_directory)
         evaluated_fixed_runs = filter_for_recent_results(project_name=project_name, stats_directories=fixed_stats_directories)
         fixed_raw_stats = evaluate_directories(project_name=fixed_project_name, results_directory=fixed_results_directory, directories=evaluated_fixed_runs)
-
+        
         # obtain mean/st dev
         final_stats = generate_report_stats(stat_values=raw_stats)
         final_fixed_stats = generate_report_stats(stat_values=fixed_raw_stats)

artifacts/output/rq4.tex

@@ -0,0 +1,44 @@
+\begin{tabular}{lllll}
+N & Property & Vanilla & Improved & Overhead \\
+\hline
+\multicolumn{5}{c}{convex} \\
+\hline
+1 & message & 45.22 ± 3.36 & 42.80 ± 2.70 & 0.95 \\
+2 & data & 53.02 ± 1.99 & 53.92 ± 0.00 & 1.02 \\
+3 & primitive & 48.52 ± 0.88 & 50.12 ± 0.00 & 1.03 \\
+\textbf{} & \textbf{Average} & \textbf{48.92} & \textbf{48.95} & \textbf{1.00} \\
+\hline
+\multicolumn{5}{c}{jflex} \\
+\hline
+1 & remove & 41.50 ± 6.41 & 31.56 ± 5.16 & 0.76 \\
+2 & addSingleton & 44.89 ± 0.77 & 51.57 ± 0.79 & 1.15 \\
+3 & contains & 43.82 ± 2.12 & 42.61 ± 1.68 & 0.97 \\
+4 & addSet & 44.56 ± 0.94 & 52.82 ± 0.87 & 1.19 \\
+5 & add & 43.06 ± 1.84 & 43.10 ± 1.80 & 1.00 \\
+6 & addString & 44.93 ± 1.64 & 51.53 ± 1.59 & 1.15 \\
+7 & addSingle & 44.40 ± 2.21 & 53.14 ± 1.53 & 1.20 \\
+\textbf{} & \textbf{Average} & \textbf{43.88} & \textbf{46.62} & \textbf{1.06} \\
+\hline
+\multicolumn{5}{c}{mph-table} \\
+\hline
+1 & list & 13.49 ± 0.98 & 13.19 ± 0.94 & 0.98 \\
+2 & optionals & 12.75 ± 0.76 & 13.17 ± 1.08 & 1.03 \\
+3 & short & 12.68 ± 0.67 & 13.16 ± 0.58 & 1.04 \\
+4 & int & 13.09 ± 0.81 & 12.79 ± 1.00 & 0.98 \\
+5 & pair & 12.76 ± 0.81 & 13.16 ± 1.09 & 1.03 \\
+6 & byte & 12.57 ± 0.79 & 12.85 ± 1.04 & 1.02 \\
+7 & long & 12.80 ± 0.75 & 13.04 ± 0.91 & 1.02 \\
+8 & string & 12.73 ± 0.60 & 12.99 ± 0.97 & 1.02 \\
+9 & list* & nan & 27.19 ± 3.60 & nan \\
+\textbf{} & \textbf{Average} & \textbf{12.86} & \textbf{14.62} & \textbf{1.01} \\
+\hline
+\multicolumn{5}{c}{rpki-commons} \\
+\hline
+1 & claiming & 25.33 ± 2.37 & nan & nan \\
+2 & aspa & 23.42 ± 1.72 & nan & nan \\
+3 & resources & 24.18 ± 1.60 & 26.14 ± 0.28 & 1.08 \\
+4 & roa & 24.29 ± 3.24 & nan & nan \\
+5 & manifest & 23.41 ± 2.44 & nan & nan \\
+6 & loose & 25.48 ± 2.62 & nan & nan \\
+\textbf{} & \textbf{Average} & \textbf{24.35} & \textbf{26.14} & \textbf{1.08} \\
+\end{tabular}