Merge pull request #706 from amd/dev

Merge from AMD

Author: arjunsuresh

automation/script/docker.py

@@ -173,7 +173,8 @@ def dockerfile(self_module, input_params):
         comments = []
 
     # Push Docker image if specified
-    if input_params.get('docker_push_image') in [True, 'True', 'yes']:
+    if str(input_params.get('docker_push_image')
+           ).lower() in ['true', 'yes', '1']:
         env['MLC_DOCKER_PUSH_IMAGE'] = 'yes'
 
     dockerfile_env = docker_inputs.get('env', {})
@@ -191,6 +192,9 @@ def dockerfile(self_module, input_params):
         'quiet': True, 'real_run': True
     }
 
+    if docker_inputs.get('mlc_repo_path', '') != '':
+        mlc_docker_input['mlc_repo_path'] = docker_inputs['mlc_repo_path']
+
     docker_v = False
     docker_s = False
     if is_true(input_params.get(

automation/script/docker_utils.py

@@ -136,7 +136,7 @@ def prepare_docker_inputs(input_params, docker_settings,
     """
 
     keys = [
-        "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version",
+        "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version", "mlc_repo_path",
         "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", "image_name",
         "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path", "user", "env", "build_env"
     ]

automation/script/help.py

@@ -26,12 +26,14 @@ def display_help(self_module, input_params):
     generic_inputs = self_module.input_flags_converted_to_env
 
     # Step 2: Search for scripts
-    search_result = self_module.search(input_params.copy())
-    if search_result['return'] > 0:
-        return search_result
 
-    scripts_list = search_result['list']
-    if not scripts_list:
+    r = self_module._select_script(input_params)
+    if r['return'] > 0:
+        return r
+
+    script = r['script']
+
+    if not script:
 
         print("")
         print("Please use script tags or alias/uid to get help for a specific script")
@@ -41,17 +43,14 @@ def display_help(self_module, input_params):
         print_input_descriptions(generic_inputs)
 
     else:
-        # Step 4: Iterate over scripts and generate help output
-        for script in sorted(
-                scripts_list, key=lambda x: x.meta.get('alias', '')):
-            metadata = script.meta
-            script_path = script.path
-            print_script_help(
-                metadata,
-                script_path,
-                generic_inputs,
-                env,
-                self_module)
+        metadata = script.meta
+        script_path = script.path
+        print_script_help(
+            metadata,
+            script_path,
+            generic_inputs,
+            env,
+            self_module)
 
     return {'return': 0}
 
@@ -146,22 +145,28 @@ def print_input_descriptions(input_descriptions):
     if not input_descriptions:
         print("\tNo inputs")
 
-    for key in input_descriptions:
+    for key in sorted(input_descriptions):
         field = input_descriptions[key]
         env_key = field.get('env_key', f"""MLC_TMP_{key.upper()}""")
         desc = field.get('desc')
-        default = field.get('default', 'None')
+        default = field.get('default', None)
         choices = field.get("choices", "")
         dtype = infer_type(field)
+
+        line = []
+
         # Use .ljust(15) to ensure the key occupies 15 characters minimum
-        print(f"\t--{key.ljust(26)}: maps to --env.{env_key}")
-        if desc:
-            print(f"\t{' '.ljust(30)}Desc: {desc}")
-        print(f"\t{' '.ljust(30)}Default: {default}")
+        line.append(f"--{key.ljust(26)}: maps to --env.{env_key}")
+        if default:
+            line.append(f"{' '.ljust(30)}Default: {default}")
+        # if dtype:
+        #    line.append(f"{' '.ljust(30)}Type: {dtype}")
         if choices:
-            print(f"\t{' '.ljust(30)}Choices: {choices}")
-        if dtype:
-            print(f"\t{' '.ljust(30)}Type: {dtype}")
+            line.append(f"{' '.ljust(30)}Choices: {choices}")
+        if desc:
+            line.append(f"{' '.ljust(30)}Desc: {desc}")
+
+        print("\t" + "\t\n\t".join(line))
         print("")
 
 

automation/script/module.py

@@ -251,6 +251,10 @@ def _run(self, i):
         import time
         import shutil
 
+        # Check if has --help
+        if i.get('help', False):
+            return self.help(i)
+
         # Check if save input/output to file
         repro = i.get('repro', False)
         repro_prefix = ''
@@ -788,10 +792,6 @@ def _run(self, i):
         env['MLC_TMP_CURRENT_SCRIPT_REPO_PATH'] = script_repo_path
         env['MLC_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX'] = script_repo_path_with_prefix
 
-        # Check if has --help
-        if i.get('help', False):
-            return self.help(i)
-
         run_state['script_id'] = meta['alias'] + "," + meta['uid']
         run_state['script_tags'] = script_tags
         run_state['script_variation_tags'] = variation_tags

script/build-dockerfile/customize.py

@@ -87,8 +87,10 @@ def preprocess(i):
                     "Dockerfile")))
         os.makedirs(build_context_dir, exist_ok=True)
 
+        repo_name = os.path.basename(mlc_repo_path)
         # Create mlc_repo directory relative to the build context
-        repo_build_context_path = os.path.join(build_context_dir, "mlc_repo")
+        repo_build_context_path = os.path.join(
+            build_context_dir, "mlc_repo", repo_name)
 
         # Remove existing directory if it exists
         if os.path.exists(repo_build_context_path):
@@ -309,8 +311,8 @@ def preprocess(i):
 
     docker_use_virtual_python = env.get('MLC_DOCKER_USE_VIRTUAL_PYTHON', "yes")
     if not is_false(docker_use_virtual_python):
-        f.write('RUN {} -m venv $HOME/venv/mlc'.format(python) + " " + EOL)
-        f.write('ENV PATH="$HOME/venv/mlc/bin:$PATH"' + EOL)
+        f.write('RUN {} -m venv $HOME/venv/mlcflow'.format(python) + " " + EOL)
+        f.write('ENV PATH="$HOME/venv/mlcflow/bin:$PATH"' + EOL)
     # f.write('RUN . /opt/venv/mlc/bin/activate' + EOL)
 
     f.write('ENV PATH="$PATH:$HOME/.local/bin"' + EOL)
@@ -328,22 +330,27 @@ def preprocess(i):
         EOL)
 
     f.write(EOL + '# Download MLC repo for scripts' + EOL)
-    pat = env.get('MLC_GH_TOKEN')
-    if pat:
+    pat = env.get('MLC_GH_TOKEN', '')
+
+    if pat != '':
         token_string = f" --pat={pat}"
     else:
         token_string = ""
 
     if use_copy_repo:
-        docker_repo_dest = "$HOME/MLC/repos/mlcommons@mlperf-automations"
+        repo_name = os.path.basename(relative_repo_path)
+        docker_repo_dest = f"$HOME/MLC/repos/{repo_name}"
         f.write(
-            f'COPY --chown=mlcuser:mlc {relative_repo_path} {docker_repo_dest}' +
+            f'COPY --chown={docker_user}:{docker_group} {relative_repo_path} {docker_repo_dest}' +
             EOL)
 
         f.write(EOL + '# Register MLC repository' + EOL)
         f.write(
-            'RUN mlc pull repo --url={} {token_string} --quiet'.format(docker_repo_dest) +
+            'RUN mlc add repo {} --quiet'.format(docker_repo_dest) +
             EOL)
+        # f.write(
+        #    'RUN mlc pull repo --url={} {} --quiet'.format(docker_repo_dest, token_string) +
+        #    EOL)
         f.write(EOL)
 
     else:

script/build-dockerfile/meta.yaml

@@ -45,6 +45,7 @@ input_mapping:
   gh_token: MLC_GH_TOKEN
   image_repo: MLC_DOCKER_IMAGE_REPO
   image_tag: MLC_DOCKER_IMAGE_TAG
+  mlc_repo_path: MLC_REPO_PATH
   package_manager_update_cmd: MLC_PACKAGE_MANAGER_UPDATE_CMD
   pip_extra_flags: MLC_DOCKER_PIP_INSTALL_EXTRA_FLAGS
   post_file: DOCKER_IMAGE_POST_FILE