updated priority to frequency

Signed-off-by: Joshua Rosenkranz <jmrosenk@us.ibm.com>

Author: JRosenkranz

tests/models/test_decoders.py

@@ -56,7 +56,7 @@
     [LLAMA_3p1_8B_INSTRUCT, GRANITE_3p2_8B_INSTRUCT, GRANITE_20B_CODE_INSTRUCT_8K, LLAMA_3p1_70B_INSTRUCT],
 )
 model_configuration_path = os.environ.get("FMS_TEST_SHAPES_FROM_MODEL_CONFIGURATION", "")
-model_configuration_priority = os.environ.get("FMS_TEST_SHAPES_FROM_MODEL_CONFIGURATION_PRIORITY", "0")
+model_configuration_frequency = os.environ.get("FMS_TEST_SHAPES_FROM_MODEL_CONFIGURATION_FREQUENCY", "0")
 
 # for validation level 1, the default is a failure rate of 1%
 # set this environment variable if you would like to relax that threshold
@@ -145,12 +145,12 @@
     print("ignoring FMS_TEST_SHAPES_COMMON_MODEL_PATHS, FMS_TEST_SHAPES_USE_MICRO_MODELS as configuration will be set by FMS_TEST_SHAPES_FROM_MODEL_CONFIGURATION")
     USE_MICRO_MODELS = False
     common_model_paths = []
-    priority = int(model_configuration_priority)
+    frequency = int(model_configuration_frequency)
     with open(model_configuration_path, 'r') as f:
         for line in f:
             try:
                 model_config = json.loads(line)
-                if model_config["priority"] <= priority:
+                if model_config["frequency"] <= frequency:
                     common_model_paths.append(model_config["model_id"])
                     # assume fullsize models
                     fail_thresholds[(model_config["model_id"], USE_MICRO_MODELS)] = (model_config["ce"], model_config["mean_diff"])

tests/resources/test_decoders_config.json

@@ -1,3 +1,3 @@
-{"model_id": "mistralai/Mistral-7B-Instruct-v0.3", "ce": 2.8364005851745624, "mean_diff": 0.0007839603102183846, "priority": 2}
+{"model_id": "mistralai/Mistral-7B-Instruct-v0.3", "ce": 2.8364005851745624, "mean_diff": 0.0007839603102183846, "frequency": 2}
 /* FIXME: proper mean_diff, currently using from granite 3.1 8b instruct */
-{"model_id": "ibm-granite/granite-guardian-3.1-8b", "ce": 2.493684446811673, "mean_diff": 0.0005767398688476533, "priority": 1}
+{"model_id": "ibm-granite/granite-guardian-3.1-8b", "ce": 2.493684446811673, "mean_diff": 0.0005767398688476533, "frequency": 1}