Skip to content

Commit 4f595fa

Browse files
authored
Changes to support new submission directory structure (#689)
1 parent 797fca2 commit 4f595fa

File tree

4 files changed

+26
-74
lines changed

4 files changed

+26
-74
lines changed

.github/workflows/test-mlperf-inference-submission-generation.yml

Lines changed: 9 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -29,42 +29,13 @@ jobs:
2929
python-version: [ "3.12" ]
3030
division: ["closed", "open", "closed-open"]
3131
category: ["datacenter", "edge"]
32-
round: ["5.0", "5.1"]
32+
round: ["5.1"]
3333
folder_name: ["closed", "open"]
3434
action: ["run", "docker"]
3535
exclude:
3636
- os: macos-latest
3737
- os: windows-latest
3838
- category: "edge"
39-
include:
40-
- os: ubuntu-latest
41-
python-version: "3.12"
42-
division: "closed"
43-
category: "edge"
44-
round: "5.0"
45-
folder_name: "closed-edge"
46-
action: "run"
47-
- os: ubuntu-latest
48-
python-version: "3.12"
49-
division: "closed"
50-
category: "edge"
51-
round: "5.0"
52-
folder_name: "closed-edge"
53-
action: "docker"
54-
- os: ubuntu-latest
55-
python-version: "3.12"
56-
division: "open"
57-
category: "edge"
58-
round: "5.0"
59-
folder_name: "closed-edge"
60-
action: "run"
61-
- os: ubuntu-latest
62-
python-version: "3.12"
63-
division: "open"
64-
category: "edge"
65-
round: "5.0"
66-
folder_name: "closed-edge"
67-
action: "docker"
6839
steps:
6940
- uses: actions/checkout@v3
7041
- name: Set up Python ${{ matrix.python-version }}
@@ -87,13 +58,13 @@ jobs:
8758
fi
8859
- name: Pull repo where test cases are uploaded
8960
run: |
90-
git clone -b submission-generation-examples https://github.com/mlcommons/inference.git submission_generation_examples
61+
git clone -b new-sub-dir https://github.com/anandhu-eng/inference.git submission_generation_examples
9162
- name: Run Submission Generation - round-${{ matrix.round }}${{ matrix.folder_name }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }}
9263
run: |
9364
if [ "${{ matrix.folder_name }}" == "closed" ]; then
9465
description="Test submission - contains closed edge and datacenter"
95-
elif [ "${{ matrix.folder_name }}" == "closed-power" ]; then
96-
description="Test submission - contains closed-power edge and datacenter results"
66+
# elif [ "${{ matrix.folder_name }}" == "closed-power" ]; then
67+
# description="Test submission - contains closed-power edge and datacenter results"
9768
fi
9869
# Dynamically set the log group to simulate a dynamic step name
9970
echo "::group::$description"
@@ -104,11 +75,12 @@ jobs:
10475
fi
10576
10677
if [ -n "${{ inputs.repo-url }}" ]; then
107-
mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags
108-
mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags
78+
mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --extra_checker_args="--skip-extra-accuracy-files-check" $docker_tags
79+
mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --extra_checker_args="--skip-extra-accuracy-files-check" $docker_tags
80+
10981
else
110-
mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $docker_tags
111-
mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags
82+
mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --extra_checker_args="--skip-extra-accuracy-files-check" --quiet $docker_tags
83+
mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --extra_checker_args="--skip-extra-accuracy-files-check" $docker_tags
11284
fi
11385
11486
echo "::endgroup::"

script/generate-mlperf-inference-submission/customize.py

Lines changed: 14 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -309,16 +309,11 @@ def generate_submission(env, state, inp, submission_division, logger):
309309
sub_res = new_res
310310

311311
submission_path = os.path.join(path_submission, "results", sub_res)
312-
measurement_path = os.path.join(
313-
path_submission, "measurements", sub_res)
314-
compliance_path = os.path.join(path_submission, "compliance", sub_res)
315312
system_path = os.path.join(path_submission, "systems")
316313
submission_system_path = system_path
317314

318315
if not os.path.isdir(submission_path):
319316
os.makedirs(submission_path)
320-
if not os.path.isdir(measurement_path):
321-
os.makedirs(measurement_path)
322317
if not os.path.isdir(submission_system_path):
323318
os.makedirs(submission_system_path)
324319
system_file = os.path.join(submission_system_path, sub_res + ".json")
@@ -342,8 +337,6 @@ def generate_submission(env, state, inp, submission_division, logger):
342337
platform_info_file = None
343338
result_model_path = os.path.join(result_path, model)
344339
submission_model_path = os.path.join(submission_path, model)
345-
measurement_model_path = os.path.join(measurement_path, model)
346-
compliance_model_path = os.path.join(compliance_path, model)
347340
code_model_path = os.path.join(code_path, model)
348341
scenarios = [
349342
f for f in os.listdir(result_model_path) if not os.path.isfile(
@@ -367,10 +360,6 @@ def generate_submission(env, state, inp, submission_division, logger):
367360
result_model_path, scenario)
368361
submission_scenario_path = os.path.join(
369362
submission_model_path, scenario)
370-
measurement_scenario_path = os.path.join(
371-
measurement_model_path, scenario)
372-
compliance_scenario_path = os.path.join(
373-
compliance_model_path, scenario)
374363

375364
'''
376365
if duplicate and scenario == 'singlestream':
@@ -445,20 +434,11 @@ def generate_submission(env, state, inp, submission_division, logger):
445434
)
446435
continue
447436

448-
if not os.path.isdir(measurement_scenario_path):
449-
os.makedirs(measurement_scenario_path)
450-
451437
for mode in modes:
452438
result_mode_path = os.path.join(result_scenario_path, mode)
453439
submission_mode_path = os.path.join(
454440
submission_scenario_path, mode)
455-
submission_measurement_path = measurement_scenario_path
456-
submission_compliance_path = os.path.join(
457-
compliance_scenario_path, mode)
458-
if mode.startswith("TEST"):
459-
submission_results_path = submission_compliance_path
460-
else:
461-
submission_results_path = submission_mode_path
441+
submission_results_path = submission_mode_path
462442
if os.path.exists(submission_results_path):
463443
shutil.rmtree(submission_results_path)
464444

@@ -492,10 +472,10 @@ def generate_submission(env, state, inp, submission_division, logger):
492472

493473
shutil.copy(
494474
analyzer_settings_file, os.path.join(
495-
submission_measurement_path, "analyzer_table.md"))
475+
submission_scenario_path, "analyzer_table.md"))
496476
shutil.copy(
497477
power_settings_file, os.path.join(
498-
submission_measurement_path, "power_settings.md"))
478+
submission_scenario_path, "power_settings.md"))
499479

500480
result_ranging_path = os.path.join(
501481
result_mode_path, 'ranging')
@@ -563,14 +543,14 @@ def generate_submission(env, state, inp, submission_division, logger):
563543
if os.path.exists(user_conf_path):
564544
shutil.copy(
565545
user_conf_path, os.path.join(
566-
measurement_scenario_path, 'user.conf'))
546+
submission_scenario_path, 'user.conf'))
567547
else:
568548
user_conf_path = os.path.join(
569549
result_mode_path, "user.conf")
570550
if os.path.exists(user_conf_path):
571551
shutil.copy(
572552
user_conf_path, os.path.join(
573-
submission_measurement_path, 'user.conf'))
553+
submission_scenario_path, 'user.conf'))
574554
else:
575555
if mode.lower() == "performance":
576556
return {
@@ -579,7 +559,7 @@ def generate_submission(env, state, inp, submission_division, logger):
579559
# First check for measurements directory in scenario folder
580560
measurements_json_path = os.path.join(
581561
result_scenario_path, "measurements.json")
582-
target_measurement_json_path = measurement_scenario_path
562+
target_measurement_json_path = submission_scenario_path
583563

584564
if not os.path.exists(measurements_json_path):
585565
measurements_json_path = os.path.join(
@@ -668,7 +648,7 @@ def generate_submission(env, state, inp, submission_division, logger):
668648
shutil.copy(
669649
os.path.join(
670650
result_mode_path, f), os.path.join(
671-
submission_measurement_path, f))
651+
submission_scenario_path, f))
672652
if f == "system_info.txt" and not platform_info_file:
673653
# the first found system_info.txt will be taken as platform info file for a specific model to be placed in
674654
# measurements-model folder when generating
@@ -679,7 +659,7 @@ def generate_submission(env, state, inp, submission_division, logger):
679659
shutil.copy(
680660
os.path.join(
681661
result_mode_path, f), os.path.join(
682-
submission_measurement_path, mode + "_" + f))
662+
submission_scenario_path, mode + "_" + f))
683663

684664
if mode == "accuracy":
685665
if os.path.exists(os.path.join(
@@ -705,13 +685,12 @@ def generate_submission(env, state, inp, submission_division, logger):
705685
result_scenario_path, "system_info.txt")):
706686
shutil.copy(
707687
os.path.join(
708-
result_scenario_path, "system_info.txt"), os.path.join(
709-
submission_measurement_path, f))
688+
result_scenario_path, "system_info.txt"), submission_scenario_path)
710689
platform_info_file = os.path.join(
711690
result_scenario_path, "system_info.txt")
712691

713692
readme_file = os.path.join(
714-
submission_measurement_path, "README.md")
693+
submission_scenario_path, "README.md")
715694
if not os.path.exists(readme_file):
716695
with open(readme_file, mode='w') as f:
717696
f.write("TBD") # create an empty README
@@ -741,7 +720,7 @@ def generate_submission(env, state, inp, submission_division, logger):
741720
shutil.copy(
742721
sys_info_file,
743722
os.path.join(
744-
measurement_model_path,
723+
submission_model_path,
745724
"system_info.txt"))
746725

747726
# Copy system_info.txt to the submission measurements folder if any
@@ -757,14 +736,14 @@ def generate_submission(env, state, inp, submission_division, logger):
757736
shutil.copy(
758737
sys_info_file,
759738
os.path.join(
760-
measurement_path,
739+
submission_path,
761740
"system_info.txt"))
762741
else:
763742
if env.get('MLC_GET_PLATFORM_DETAILS', '') == "yes":
764743
mlc_input = {'action': 'run',
765744
'automation': 'script',
766745
'tags': 'get,platform,details',
767-
'env': {'MLC_PLATFORM_DETAILS_FILE_PATH': os.path.join(measurement_path, "system_info.txt")},
746+
'env': {'MLC_PLATFORM_DETAILS_FILE_PATH': os.path.join(submission_path, "system_info.txt")},
768747
'quiet': True
769748
}
770749
mlc = i['automation'].action_object
@@ -779,7 +758,7 @@ def generate_submission(env, state, inp, submission_division, logger):
779758

780759
logger.info(tabulate(result_table, headers=headers, tablefmt="pretty"))
781760

782-
sut_readme_file = os.path.join(measurement_path, "README.md")
761+
sut_readme_file = os.path.join(submission_path, "README.md")
783762
with open(sut_readme_file, mode='w') as f:
784763
f.write(tabulate(result_table, headers=headers, tablefmt="github"))
785764

script/generate-mlperf-inference-submission/meta.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ input_mapping:
6666
dashboard_wb_project: MLC_MLPERF_DASHBOARD_WANDB_PROJECT
6767
device: MLC_MLPERF_DEVICE
6868
division: MLC_MLPERF_SUBMISSION_DIVISION
69-
extra_checker_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARG
69+
extra_checker_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS
7070
hw_name: MLC_HW_NAME
7171
hw_notes_extra: MLC_MLPERF_SUT_HW_NOTES_EXTRA
7272
noinfer_scenario_results: MLC_MLPERF_NOINFER_SCENARIO_RESULTS

script/get-mlperf-inference-utils/mlperf_utils.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,8 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res,
188188
result['performance'] = round(performance_result_, 3)
189189

190190
if scenario != effective_scenario:
191-
inferred, inferred_result = checker.get_inferred_result(
191+
print(f"{fname} : {scenario} : {effective_scenario}")
192+
inferred, inferred_result, inferred_valid = checker.get_inferred_result(
192193
scenario, effective_scenario, performance_result, mlperf_log, config, False)
193194

194195
if has_power:

0 commit comments

Comments
 (0)