diff --git a/sagemaker-core/src/sagemaker/core/processing.py b/sagemaker-core/src/sagemaker/core/processing.py index 9dec7060bc..e1725a3c81 100644 --- a/sagemaker-core/src/sagemaker/core/processing.py +++ b/sagemaker-core/src/sagemaker/core/processing.py @@ -332,7 +332,8 @@ def run( is built with :class:`~sagemaker.workflow.pipeline_context.PipelineSession`. However, the value of `TrialComponentDisplayName` is honored for display in Studio. kms_key (str): The ARN of the KMS key that is used to encrypt the - user code file (default: None). + user code file. If not provided, the processor's configured + ``output_kms_key`` is used (default: None). Returns: None or pipeline step arguments in case the Processor instance is built with :class:`~sagemaker.workflow.pipeline_context.PipelineSession` @@ -343,6 +344,12 @@ def run( raise ValueError("""Logs can only be shown if wait is set to True. Please either set wait to True or set logs to False.""") + # When no explicit code-encryption key is given, fall back to the + # configured output KMS key so the uploaded code and job outputs are + # encrypted with the same key. + if kms_key is None: + kms_key = self.output_kms_key + normalized_inputs, normalized_outputs = self._normalize_args( job_name=job_name, arguments=arguments, @@ -903,7 +910,8 @@ def run( is built with :class:`~sagemaker.workflow.pipeline_context.PipelineSession`. However, the value of `TrialComponentDisplayName` is honored for display in Studio. kms_key (str): The ARN of the KMS key that is used to encrypt the - user code file (default: None). + user code file. If not provided, the processor's configured + ``output_kms_key`` is used (default: None). Returns: None or pipeline step arguments in case the Processor instance is built with :class:`~sagemaker.workflow.pipeline_context.PipelineSession` @@ -914,7 +922,7 @@ def run( inputs=inputs, outputs=outputs, code=code, - kms_key=kms_key, + kms_key=kms_key if kms_key is not None else self.output_kms_key, ) experiment_config = check_and_get_run_experiment_config(experiment_config) @@ -1481,6 +1489,7 @@ def _pack_and_upload_code( entry_point, source_dir, install_requirements_dir, + requirements=requirements, ) return s3_runproc_sh, inputs, job_name @@ -1526,13 +1535,14 @@ def _create_and_upload_runproc( entry_point=None, source_dir=None, install_requirements_dir=None, + requirements=None, ): """Create runproc shell script and upload to S3 bucket.""" from sagemaker.core.workflow.utilities import _pipeline_config, hash_object if _pipeline_config and _pipeline_config.pipeline_name: runproc_file_str = self._generate_framework_script( - user_script, entry_point, source_dir, install_requirements_dir + user_script, entry_point, source_dir, install_requirements_dir, requirements ) runproc_file_hash = hash_object(runproc_file_str) s3_uri = s3.s3_path_join( @@ -1551,7 +1561,7 @@ def _create_and_upload_runproc( else: s3_runproc_sh = s3.S3Uploader.upload_string_as_file_body( self._generate_framework_script( - user_script, entry_point, source_dir, install_requirements_dir + user_script, entry_point, source_dir, install_requirements_dir, requirements ), desired_s3_uri=entrypoint_s3_uri, kms_key=kms_key, @@ -1560,20 +1570,36 @@ def _create_and_upload_runproc( return s3_runproc_sh + @staticmethod + def _requirements_file_in_container(requirements: Optional[str]) -> str: + """Return the requirements path as it appears inside the extracted source bundle. + + ``requirements`` is documented as relative to ``source_dir`` and ``_package_code`` + preserves the directory layout, so a relative path (``reqs/cpu.txt``) is kept. An + absolute path cannot be located inside the bundle, so only its basename is used. + """ + if not requirements: + return "requirements.txt" + if os.path.isabs(requirements): + return os.path.basename(requirements) + return os.path.normpath(requirements).replace(os.sep, "/") + def _generate_framework_script( self, user_script: str, entry_point: str = None, source_dir: str = None, install_requirements_dir: str = None, + requirements: str = None, ) -> str: """Generate the framework entrypoint file (as text) for a processing job.""" if entry_point: return self._generate_custom_framework_script( - user_script, entry_point, source_dir, install_requirements_dir + user_script, entry_point, source_dir, install_requirements_dir, requirements ) install_requirements_dir = install_requirements_dir or self._SOURCE_CODE_CONTAINER_DIR + requirements_file = self._requirements_file_in_container(requirements) return dedent("""\ #!/bin/bash @@ -1597,16 +1623,17 @@ def _generate_framework_script( exit 1 fi - if [[ -f 'requirements.txt' ]]; then + if [[ -f '{requirements_file}' ]]; then # Some py3 containers has typing, which may breaks pip install pip uninstall --yes typing - python3 {install_requirements_dir}/install_requirements.py requirements.txt + python3 {install_requirements_dir}/install_requirements.py {requirements_file} fi {entry_point_command} {entry_point} "$@" """).format( install_requirements_dir=install_requirements_dir, + requirements_file=requirements_file, entry_point_command=" ".join(self.command), entry_point=user_script, ) @@ -1617,6 +1644,7 @@ def _generate_custom_framework_script( entry_point: str, source_dir: str = None, install_requirements_dir: str = None, + requirements: str = None, ) -> str: """Generate a custom framework script with a user-provided entrypoint embedded. @@ -1630,6 +1658,8 @@ def _generate_custom_framework_script( is relative, it will be combined with source_dir. install_requirements_dir (str): Container directory that holds ``install_requirements.py`` (default: the extracted source code dir). + requirements (str): Path to the requirements file relative to source_dir + (default: ``requirements.txt``). Returns: str: The generated script content @@ -1639,6 +1669,7 @@ def _generate_custom_framework_script( # source bundle on the container. if self._is_s3_uri(source_dir): install_requirements_dir = install_requirements_dir or self._SOURCE_CODE_CONTAINER_DIR + requirements_file = self._requirements_file_in_container(requirements) return dedent("""\ #!/bin/bash @@ -1655,9 +1686,9 @@ def _generate_custom_framework_script( exit 1 fi - if [[ -f 'requirements.txt' ]]; then + if [[ -f '{requirements_file}' ]]; then pip uninstall --yes typing - python3 {install_requirements_dir}/install_requirements.py requirements.txt + python3 {install_requirements_dir}/install_requirements.py {requirements_file} fi # Execute custom entrypoint @@ -1667,6 +1698,7 @@ def _generate_custom_framework_script( {entry_point_command} {user_script} "$@" """).format( install_requirements_dir=install_requirements_dir, + requirements_file=requirements_file, entry_point=entry_point, entry_point_command=" ".join(self.command), user_script=user_script, diff --git a/sagemaker-core/src/sagemaker/core/spark/processing.py b/sagemaker-core/src/sagemaker/core/spark/processing.py index 6f959ea4bb..f874e94def 100644 --- a/sagemaker-core/src/sagemaker/core/spark/processing.py +++ b/sagemaker-core/src/sagemaker/core/spark/processing.py @@ -39,6 +39,7 @@ from sagemaker.core import s3 from sagemaker.core.local.image import _ecr_login_if_needed, _pull_image from sagemaker.core.processing import ProcessingInput, ProcessingOutput, ScriptProcessor +from sagemaker.core.shapes import ProcessingS3Input, ProcessingS3Output from sagemaker.core.s3 import S3Uploader from sagemaker.core.helper.session_helper import Session from sagemaker.core.network import NetworkConfig @@ -330,9 +331,12 @@ def _extend_processing_args(self, inputs, outputs, **kwargs): ) output = ProcessingOutput( - source=_SparkProcessorBase._spark_event_log_default_local_path, - destination=spark_event_logs_s3_uri, - s3_upload_mode="Continuous", + output_name="spark-event-logs", + s3_output=ProcessingS3Output( + s3_uri=spark_event_logs_s3_uri, + local_path=_SparkProcessorBase._spark_event_log_default_local_path, + s3_upload_mode="Continuous", + ), ) extended_outputs.append(output) @@ -444,9 +448,13 @@ def _stage_configuration(self, configuration): ) conf_input = ProcessingInput( - source=s3_uri, - destination=f"{self._conf_container_base_path}{self._conf_container_input_name}", input_name=_SparkProcessorBase._conf_container_input_name, + s3_input=ProcessingS3Input( + s3_uri=s3_uri, + local_path=f"{self._conf_container_base_path}{self._conf_container_input_name}", + s3_data_type="S3Prefix", + s3_input_mode="File", + ), ) return conf_input @@ -473,6 +481,11 @@ def _stage_submit_deps(self, submit_deps, input_channel_name): ) if not input_channel_name: raise ValueError("input_channel_name value may not be empty.") + if not isinstance(submit_deps, (list, tuple)): + raise ValueError( + f"submit_deps must be a list of one or more paths, but got " + f"{type(submit_deps).__name__}. {self._submit_deps_error_message}" + ) use_input_channel = False spark_opt_s3_uris = [] @@ -544,15 +557,20 @@ def _stage_submit_deps(self, submit_deps, input_channel_name): # them to the Spark container and form the spark-submit option from a # combination of S3 URIs and container's local input path if use_input_channel: + input_channel_local_path = f"{self._conf_container_base_path}{input_channel_name}" input_channel = ProcessingInput( - source=input_channel_s3_uri, - destination=f"{self._conf_container_base_path}{input_channel_name}", input_name=input_channel_name, + s3_input=ProcessingS3Input( + s3_uri=input_channel_s3_uri, + local_path=input_channel_local_path, + s3_data_type="S3Prefix", + s3_input_mode="File", + ), ) spark_opt = ( - Join(on=",", values=spark_opt_s3_uris + [input_channel.destination]) + Join(on=",", values=spark_opt_s3_uris + [input_channel_local_path]) if spark_opt_s3_uris_has_pipeline_var - else ",".join(spark_opt_s3_uris + [input_channel.destination]) + else ",".join(spark_opt_s3_uris + [input_channel_local_path]) ) # If no local files were uploaded, form the spark-submit option from a list of S3 URIs else: diff --git a/sagemaker-core/tests/data/spark/code/python/hello_py_spark/hello_py_spark_app.py b/sagemaker-core/tests/data/spark/code/python/hello_py_spark/hello_py_spark_app.py new file mode 100644 index 0000000000..c4f7c12dfe --- /dev/null +++ b/sagemaker-core/tests/data/spark/code/python/hello_py_spark/hello_py_spark_app.py @@ -0,0 +1,55 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import argparse +import time + +from pyspark.sql import SparkSession, SQLContext +from pyspark.sql.functions import udf +from pyspark.sql.types import IntegerType + +# Import local module to test spark-submit--py-files dependencies +import hello_py_spark_udfs as udfs + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="inputs and outputs") + parser.add_argument("--input", type=str, help="path to input data") + parser.add_argument("--output", required=False, type=str, help="path to output data") + args = parser.parse_args() + spark = SparkSession.builder.appName("SparkContainerTestApp").getOrCreate() + sqlContext = SQLContext(spark.sparkContext) + + # Load test data set + inputPath = args.input + salesDF = spark.read.json(inputPath) + salesDF.printSchema() + + salesDF.createOrReplaceTempView("sales") + topDF = spark.sql("SELECT date, sale FROM sales WHERE sale > 750") + # Show the first 20 rows of the dataframe + topDF.show() + time.sleep(60) + + # Calculate average sales by date + averageSalesPerDay = salesDF.groupBy("date").avg().collect() + print(averageSalesPerDay) + + outputPath = args.output + + # Define a UDF that doubles an integer column + # The UDF function is imported from local module to test spark-submit--py-files dependencies + double_udf_int = udf(udfs.double_x, IntegerType()) + + # Save transformed data set to disk + salesDF.select("date", "sale", double_udf_int("sale").alias("sale_double")).write.json( + outputPath + ) diff --git a/sagemaker-core/tests/data/spark/code/python/hello_py_spark/hello_py_spark_udfs.py b/sagemaker-core/tests/data/spark/code/python/hello_py_spark/hello_py_spark_udfs.py new file mode 100644 index 0000000000..7c2f8e02a2 --- /dev/null +++ b/sagemaker-core/tests/data/spark/code/python/hello_py_spark/hello_py_spark_udfs.py @@ -0,0 +1,2 @@ +def double_x(x): + return x + x diff --git a/sagemaker-core/tests/data/spark/files/data.jsonl b/sagemaker-core/tests/data/spark/files/data.jsonl new file mode 100644 index 0000000000..a2e10ac774 --- /dev/null +++ b/sagemaker-core/tests/data/spark/files/data.jsonl @@ -0,0 +1,1000 @@ +{"date":"2020-01-04","sale":283} +{"date":"2020-01-06","sale":140} +{"date":"2020-01-05","sale":820} +{"date":"2020-01-04","sale":452} +{"date":"2020-01-06","sale":495} +{"date":"2020-01-04","sale":247} +{"date":"2020-01-05","sale":479} +{"date":"2020-01-04","sale":15} +{"date":"2020-01-04","sale":197} +{"date":"2020-01-04","sale":15} +{"date":"2020-01-02","sale":702} +{"date":"2020-01-07","sale":422} +{"date":"2020-01-04","sale":42} +{"date":"2020-01-01","sale":5} +{"date":"2020-01-06","sale":40} +{"date":"2020-01-01","sale":626} +{"date":"2020-01-05","sale":256} +{"date":"2020-01-06","sale":325} +{"date":"2020-01-06","sale":857} +{"date":"2020-01-04","sale":495} +{"date":"2020-01-02","sale":712} +{"date":"2020-01-02","sale":291} +{"date":"2020-01-02","sale":850} +{"date":"2020-01-01","sale":581} +{"date":"2020-01-05","sale":504} +{"date":"2020-01-07","sale":919} +{"date":"2020-01-07","sale":833} +{"date":"2020-01-03","sale":186} +{"date":"2020-01-03","sale":143} +{"date":"2020-01-07","sale":784} +{"date":"2020-01-04","sale":367} +{"date":"2020-01-07","sale":80} +{"date":"2020-01-07","sale":603} +{"date":"2020-01-01","sale":620} +{"date":"2020-01-03","sale":309} +{"date":"2020-01-02","sale":189} +{"date":"2020-01-07","sale":985} +{"date":"2020-01-05","sale":326} +{"date":"2020-01-05","sale":741} +{"date":"2020-01-04","sale":917} +{"date":"2020-01-05","sale":733} +{"date":"2020-01-05","sale":990} +{"date":"2020-01-01","sale":750} +{"date":"2020-01-06","sale":34} +{"date":"2020-01-03","sale":77} +{"date":"2020-01-03","sale":109} +{"date":"2020-01-05","sale":48} +{"date":"2020-01-05","sale":788} +{"date":"2020-01-01","sale":865} +{"date":"2020-01-05","sale":701} +{"date":"2020-01-04","sale":851} +{"date":"2020-01-05","sale":625} +{"date":"2020-01-01","sale":173} +{"date":"2020-01-07","sale":684} +{"date":"2020-01-04","sale":284} +{"date":"2020-01-05","sale":62} +{"date":"2020-01-04","sale":718} +{"date":"2020-01-02","sale":777} +{"date":"2020-01-07","sale":536} +{"date":"2020-01-04","sale":930} +{"date":"2020-01-05","sale":80} +{"date":"2020-01-02","sale":821} +{"date":"2020-01-04","sale":231} +{"date":"2020-01-05","sale":674} +{"date":"2020-01-02","sale":886} +{"date":"2020-01-06","sale":372} +{"date":"2020-01-02","sale":641} +{"date":"2020-01-06","sale":111} +{"date":"2020-01-04","sale":313} +{"date":"2020-01-07","sale":993} +{"date":"2020-01-05","sale":604} +{"date":"2020-01-06","sale":825} +{"date":"2020-01-03","sale":119} +{"date":"2020-01-01","sale":740} +{"date":"2020-01-03","sale":155} +{"date":"2020-01-02","sale":991} +{"date":"2020-01-03","sale":989} +{"date":"2020-01-06","sale":143} +{"date":"2020-01-04","sale":88} +{"date":"2020-01-04","sale":860} +{"date":"2020-01-02","sale":349} +{"date":"2020-01-05","sale":972} +{"date":"2020-01-05","sale":274} +{"date":"2020-01-05","sale":475} +{"date":"2020-01-06","sale":681} +{"date":"2020-01-02","sale":216} +{"date":"2020-01-04","sale":231} +{"date":"2020-01-05","sale":884} +{"date":"2020-01-04","sale":261} +{"date":"2020-01-06","sale":603} +{"date":"2020-01-02","sale":931} +{"date":"2020-01-04","sale":620} +{"date":"2020-01-06","sale":456} +{"date":"2020-01-07","sale":437} +{"date":"2020-01-02","sale":303} +{"date":"2020-01-03","sale":672} +{"date":"2020-01-03","sale":843} +{"date":"2020-01-01","sale":314} +{"date":"2020-01-03","sale":2} +{"date":"2020-01-05","sale":618} +{"date":"2020-01-06","sale":488} +{"date":"2020-01-01","sale":495} +{"date":"2020-01-02","sale":883} +{"date":"2020-01-02","sale":145} +{"date":"2020-01-01","sale":227} +{"date":"2020-01-06","sale":794} +{"date":"2020-01-05","sale":364} +{"date":"2020-01-06","sale":638} +{"date":"2020-01-06","sale":636} +{"date":"2020-01-05","sale":525} +{"date":"2020-01-07","sale":832} +{"date":"2020-01-01","sale":249} +{"date":"2020-01-04","sale":739} +{"date":"2020-01-07","sale":546} +{"date":"2020-01-01","sale":790} +{"date":"2020-01-01","sale":839} +{"date":"2020-01-03","sale":258} +{"date":"2020-01-01","sale":144} +{"date":"2020-01-01","sale":418} +{"date":"2020-01-02","sale":847} +{"date":"2020-01-06","sale":574} +{"date":"2020-01-05","sale":668} +{"date":"2020-01-04","sale":122} +{"date":"2020-01-07","sale":95} +{"date":"2020-01-01","sale":542} +{"date":"2020-01-01","sale":289} +{"date":"2020-01-04","sale":574} +{"date":"2020-01-03","sale":915} +{"date":"2020-01-01","sale":82} +{"date":"2020-01-03","sale":80} +{"date":"2020-01-02","sale":512} +{"date":"2020-01-01","sale":79} +{"date":"2020-01-01","sale":668} +{"date":"2020-01-01","sale":752} +{"date":"2020-01-01","sale":625} +{"date":"2020-01-01","sale":39} +{"date":"2020-01-04","sale":666} +{"date":"2020-01-03","sale":822} +{"date":"2020-01-03","sale":426} +{"date":"2020-01-06","sale":381} +{"date":"2020-01-01","sale":993} +{"date":"2020-01-04","sale":365} +{"date":"2020-01-03","sale":375} +{"date":"2020-01-07","sale":555} +{"date":"2020-01-05","sale":403} +{"date":"2020-01-03","sale":208} +{"date":"2020-01-01","sale":165} +{"date":"2020-01-07","sale":564} +{"date":"2020-01-06","sale":10} +{"date":"2020-01-05","sale":306} +{"date":"2020-01-01","sale":429} +{"date":"2020-01-05","sale":863} +{"date":"2020-01-06","sale":906} +{"date":"2020-01-02","sale":596} +{"date":"2020-01-03","sale":310} +{"date":"2020-01-01","sale":419} +{"date":"2020-01-04","sale":642} +{"date":"2020-01-03","sale":999} +{"date":"2020-01-05","sale":359} +{"date":"2020-01-05","sale":276} +{"date":"2020-01-01","sale":307} +{"date":"2020-01-04","sale":485} +{"date":"2020-01-03","sale":401} +{"date":"2020-01-07","sale":908} +{"date":"2020-01-07","sale":639} +{"date":"2020-01-06","sale":844} +{"date":"2020-01-03","sale":625} +{"date":"2020-01-04","sale":98} +{"date":"2020-01-03","sale":242} +{"date":"2020-01-04","sale":962} +{"date":"2020-01-01","sale":411} +{"date":"2020-01-02","sale":862} +{"date":"2020-01-04","sale":596} +{"date":"2020-01-04","sale":569} +{"date":"2020-01-04","sale":56} +{"date":"2020-01-04","sale":858} +{"date":"2020-01-02","sale":53} +{"date":"2020-01-04","sale":340} +{"date":"2020-01-02","sale":720} +{"date":"2020-01-01","sale":146} +{"date":"2020-01-06","sale":935} +{"date":"2020-01-04","sale":592} +{"date":"2020-01-03","sale":208} +{"date":"2020-01-06","sale":908} +{"date":"2020-01-01","sale":48} +{"date":"2020-01-05","sale":548} +{"date":"2020-01-03","sale":205} +{"date":"2020-01-04","sale":617} +{"date":"2020-01-04","sale":228} +{"date":"2020-01-04","sale":472} +{"date":"2020-01-01","sale":631} +{"date":"2020-01-05","sale":345} +{"date":"2020-01-02","sale":486} +{"date":"2020-01-06","sale":617} +{"date":"2020-01-01","sale":158} +{"date":"2020-01-01","sale":266} +{"date":"2020-01-02","sale":191} +{"date":"2020-01-01","sale":635} +{"date":"2020-01-07","sale":896} +{"date":"2020-01-06","sale":754} +{"date":"2020-01-03","sale":29} +{"date":"2020-01-01","sale":509} +{"date":"2020-01-07","sale":416} +{"date":"2020-01-06","sale":368} +{"date":"2020-01-02","sale":358} +{"date":"2020-01-03","sale":600} +{"date":"2020-01-05","sale":136} +{"date":"2020-01-05","sale":862} +{"date":"2020-01-05","sale":257} +{"date":"2020-01-06","sale":176} +{"date":"2020-01-03","sale":464} +{"date":"2020-01-01","sale":881} +{"date":"2020-01-03","sale":771} +{"date":"2020-01-01","sale":961} +{"date":"2020-01-04","sale":811} +{"date":"2020-01-05","sale":15} +{"date":"2020-01-01","sale":279} +{"date":"2020-01-06","sale":722} +{"date":"2020-01-05","sale":657} +{"date":"2020-01-06","sale":135} +{"date":"2020-01-05","sale":677} +{"date":"2020-01-02","sale":357} +{"date":"2020-01-06","sale":455} +{"date":"2020-01-03","sale":204} +{"date":"2020-01-01","sale":579} +{"date":"2020-01-07","sale":540} +{"date":"2020-01-02","sale":880} +{"date":"2020-01-04","sale":370} +{"date":"2020-01-05","sale":254} +{"date":"2020-01-07","sale":197} +{"date":"2020-01-06","sale":719} +{"date":"2020-01-03","sale":319} +{"date":"2020-01-04","sale":298} +{"date":"2020-01-06","sale":879} +{"date":"2020-01-02","sale":106} +{"date":"2020-01-04","sale":235} +{"date":"2020-01-05","sale":125} +{"date":"2020-01-02","sale":570} +{"date":"2020-01-04","sale":40} +{"date":"2020-01-07","sale":360} +{"date":"2020-01-05","sale":736} +{"date":"2020-01-05","sale":898} +{"date":"2020-01-04","sale":659} +{"date":"2020-01-03","sale":521} +{"date":"2020-01-07","sale":333} +{"date":"2020-01-04","sale":972} +{"date":"2020-01-03","sale":199} +{"date":"2020-01-07","sale":990} +{"date":"2020-01-07","sale":337} +{"date":"2020-01-04","sale":959} +{"date":"2020-01-04","sale":128} +{"date":"2020-01-04","sale":111} +{"date":"2020-01-01","sale":523} +{"date":"2020-01-07","sale":996} +{"date":"2020-01-06","sale":799} +{"date":"2020-01-01","sale":932} +{"date":"2020-01-05","sale":123} +{"date":"2020-01-06","sale":502} +{"date":"2020-01-04","sale":238} +{"date":"2020-01-04","sale":363} +{"date":"2020-01-02","sale":859} +{"date":"2020-01-07","sale":907} +{"date":"2020-01-06","sale":998} +{"date":"2020-01-07","sale":681} +{"date":"2020-01-02","sale":1} +{"date":"2020-01-05","sale":960} +{"date":"2020-01-02","sale":704} +{"date":"2020-01-07","sale":327} +{"date":"2020-01-06","sale":971} +{"date":"2020-01-03","sale":127} +{"date":"2020-01-07","sale":737} +{"date":"2020-01-05","sale":662} +{"date":"2020-01-04","sale":48} +{"date":"2020-01-01","sale":420} +{"date":"2020-01-01","sale":491} +{"date":"2020-01-06","sale":715} +{"date":"2020-01-07","sale":181} +{"date":"2020-01-07","sale":691} +{"date":"2020-01-06","sale":698} +{"date":"2020-01-05","sale":699} +{"date":"2020-01-05","sale":237} +{"date":"2020-01-01","sale":124} +{"date":"2020-01-03","sale":612} +{"date":"2020-01-05","sale":191} +{"date":"2020-01-07","sale":560} +{"date":"2020-01-07","sale":71} +{"date":"2020-01-03","sale":784} +{"date":"2020-01-02","sale":441} +{"date":"2020-01-07","sale":892} +{"date":"2020-01-07","sale":224} +{"date":"2020-01-06","sale":951} +{"date":"2020-01-04","sale":341} +{"date":"2020-01-07","sale":798} +{"date":"2020-01-02","sale":691} +{"date":"2020-01-04","sale":126} +{"date":"2020-01-05","sale":168} +{"date":"2020-01-02","sale":698} +{"date":"2020-01-01","sale":656} +{"date":"2020-01-04","sale":841} +{"date":"2020-01-07","sale":668} +{"date":"2020-01-05","sale":686} +{"date":"2020-01-04","sale":960} +{"date":"2020-01-01","sale":273} +{"date":"2020-01-05","sale":482} +{"date":"2020-01-01","sale":955} +{"date":"2020-01-01","sale":957} +{"date":"2020-01-06","sale":103} +{"date":"2020-01-07","sale":189} +{"date":"2020-01-05","sale":139} +{"date":"2020-01-03","sale":686} +{"date":"2020-01-05","sale":974} +{"date":"2020-01-03","sale":605} +{"date":"2020-01-02","sale":251} +{"date":"2020-01-05","sale":880} +{"date":"2020-01-01","sale":932} +{"date":"2020-01-03","sale":89} +{"date":"2020-01-02","sale":138} +{"date":"2020-01-06","sale":909} +{"date":"2020-01-07","sale":432} +{"date":"2020-01-01","sale":571} +{"date":"2020-01-06","sale":904} +{"date":"2020-01-02","sale":520} +{"date":"2020-01-05","sale":637} +{"date":"2020-01-04","sale":808} +{"date":"2020-01-01","sale":360} +{"date":"2020-01-01","sale":191} +{"date":"2020-01-01","sale":462} +{"date":"2020-01-03","sale":661} +{"date":"2020-01-03","sale":45} +{"date":"2020-01-03","sale":471} +{"date":"2020-01-07","sale":191} +{"date":"2020-01-04","sale":746} +{"date":"2020-01-02","sale":784} +{"date":"2020-01-03","sale":619} +{"date":"2020-01-05","sale":394} +{"date":"2020-01-03","sale":186} +{"date":"2020-01-02","sale":392} +{"date":"2020-01-03","sale":629} +{"date":"2020-01-05","sale":495} +{"date":"2020-01-06","sale":201} +{"date":"2020-01-03","sale":80} +{"date":"2020-01-05","sale":107} +{"date":"2020-01-03","sale":665} +{"date":"2020-01-05","sale":468} +{"date":"2020-01-05","sale":76} +{"date":"2020-01-05","sale":899} +{"date":"2020-01-06","sale":720} +{"date":"2020-01-04","sale":932} +{"date":"2020-01-04","sale":627} +{"date":"2020-01-05","sale":118} +{"date":"2020-01-04","sale":225} +{"date":"2020-01-01","sale":747} +{"date":"2020-01-04","sale":550} +{"date":"2020-01-02","sale":64} +{"date":"2020-01-05","sale":807} +{"date":"2020-01-07","sale":932} +{"date":"2020-01-01","sale":215} +{"date":"2020-01-05","sale":350} +{"date":"2020-01-03","sale":725} +{"date":"2020-01-05","sale":742} +{"date":"2020-01-03","sale":11} +{"date":"2020-01-03","sale":470} +{"date":"2020-01-03","sale":785} +{"date":"2020-01-06","sale":618} +{"date":"2020-01-03","sale":872} +{"date":"2020-01-07","sale":850} +{"date":"2020-01-03","sale":307} +{"date":"2020-01-05","sale":912} +{"date":"2020-01-02","sale":895} +{"date":"2020-01-05","sale":269} +{"date":"2020-01-02","sale":331} +{"date":"2020-01-01","sale":654} +{"date":"2020-01-03","sale":322} +{"date":"2020-01-07","sale":571} +{"date":"2020-01-07","sale":237} +{"date":"2020-01-04","sale":712} +{"date":"2020-01-04","sale":574} +{"date":"2020-01-03","sale":813} +{"date":"2020-01-06","sale":8} +{"date":"2020-01-07","sale":528} +{"date":"2020-01-03","sale":594} +{"date":"2020-01-04","sale":314} +{"date":"2020-01-07","sale":490} +{"date":"2020-01-06","sale":543} +{"date":"2020-01-03","sale":942} +{"date":"2020-01-06","sale":647} +{"date":"2020-01-03","sale":133} +{"date":"2020-01-02","sale":9} +{"date":"2020-01-01","sale":924} +{"date":"2020-01-07","sale":696} +{"date":"2020-01-05","sale":5} +{"date":"2020-01-04","sale":229} +{"date":"2020-01-07","sale":130} +{"date":"2020-01-02","sale":35} +{"date":"2020-01-03","sale":416} +{"date":"2020-01-06","sale":90} +{"date":"2020-01-01","sale":534} +{"date":"2020-01-02","sale":883} +{"date":"2020-01-02","sale":364} +{"date":"2020-01-01","sale":235} +{"date":"2020-01-07","sale":456} +{"date":"2020-01-05","sale":379} +{"date":"2020-01-04","sale":363} +{"date":"2020-01-07","sale":371} +{"date":"2020-01-03","sale":946} +{"date":"2020-01-04","sale":303} +{"date":"2020-01-03","sale":89} +{"date":"2020-01-02","sale":761} +{"date":"2020-01-03","sale":67} +{"date":"2020-01-04","sale":190} +{"date":"2020-01-01","sale":676} +{"date":"2020-01-06","sale":949} +{"date":"2020-01-04","sale":87} +{"date":"2020-01-04","sale":930} +{"date":"2020-01-06","sale":131} +{"date":"2020-01-01","sale":521} +{"date":"2020-01-05","sale":704} +{"date":"2020-01-02","sale":5} +{"date":"2020-01-03","sale":328} +{"date":"2020-01-04","sale":304} +{"date":"2020-01-02","sale":352} +{"date":"2020-01-06","sale":941} +{"date":"2020-01-05","sale":917} +{"date":"2020-01-06","sale":793} +{"date":"2020-01-03","sale":844} +{"date":"2020-01-02","sale":304} +{"date":"2020-01-06","sale":186} +{"date":"2020-01-04","sale":300} +{"date":"2020-01-01","sale":474} +{"date":"2020-01-01","sale":888} +{"date":"2020-01-03","sale":61} +{"date":"2020-01-03","sale":811} +{"date":"2020-01-07","sale":179} +{"date":"2020-01-01","sale":936} +{"date":"2020-01-05","sale":711} +{"date":"2020-01-06","sale":746} +{"date":"2020-01-05","sale":313} +{"date":"2020-01-07","sale":653} +{"date":"2020-01-07","sale":229} +{"date":"2020-01-05","sale":120} +{"date":"2020-01-03","sale":601} +{"date":"2020-01-06","sale":70} +{"date":"2020-01-03","sale":461} +{"date":"2020-01-05","sale":323} +{"date":"2020-01-07","sale":994} +{"date":"2020-01-03","sale":468} +{"date":"2020-01-06","sale":436} +{"date":"2020-01-01","sale":23} +{"date":"2020-01-04","sale":844} +{"date":"2020-01-05","sale":554} +{"date":"2020-01-01","sale":934} +{"date":"2020-01-06","sale":743} +{"date":"2020-01-05","sale":698} +{"date":"2020-01-02","sale":309} +{"date":"2020-01-06","sale":494} +{"date":"2020-01-04","sale":74} +{"date":"2020-01-02","sale":96} +{"date":"2020-01-05","sale":860} +{"date":"2020-01-06","sale":736} +{"date":"2020-01-01","sale":15} +{"date":"2020-01-03","sale":924} +{"date":"2020-01-07","sale":496} +{"date":"2020-01-04","sale":542} +{"date":"2020-01-03","sale":928} +{"date":"2020-01-02","sale":144} +{"date":"2020-01-02","sale":765} +{"date":"2020-01-02","sale":816} +{"date":"2020-01-02","sale":131} +{"date":"2020-01-03","sale":846} +{"date":"2020-01-04","sale":255} +{"date":"2020-01-04","sale":858} +{"date":"2020-01-05","sale":235} +{"date":"2020-01-01","sale":770} +{"date":"2020-01-05","sale":469} +{"date":"2020-01-01","sale":633} +{"date":"2020-01-07","sale":891} +{"date":"2020-01-07","sale":148} +{"date":"2020-01-03","sale":81} +{"date":"2020-01-04","sale":389} +{"date":"2020-01-02","sale":75} +{"date":"2020-01-05","sale":779} +{"date":"2020-01-07","sale":446} +{"date":"2020-01-03","sale":329} +{"date":"2020-01-07","sale":824} +{"date":"2020-01-03","sale":738} +{"date":"2020-01-02","sale":200} +{"date":"2020-01-04","sale":518} +{"date":"2020-01-04","sale":128} +{"date":"2020-01-07","sale":621} +{"date":"2020-01-07","sale":231} +{"date":"2020-01-07","sale":619} +{"date":"2020-01-04","sale":499} +{"date":"2020-01-01","sale":613} +{"date":"2020-01-07","sale":131} +{"date":"2020-01-02","sale":775} +{"date":"2020-01-03","sale":971} +{"date":"2020-01-05","sale":351} +{"date":"2020-01-06","sale":457} +{"date":"2020-01-03","sale":244} +{"date":"2020-01-03","sale":308} +{"date":"2020-01-01","sale":646} +{"date":"2020-01-01","sale":392} +{"date":"2020-01-06","sale":754} +{"date":"2020-01-07","sale":514} +{"date":"2020-01-01","sale":184} +{"date":"2020-01-04","sale":347} +{"date":"2020-01-02","sale":452} +{"date":"2020-01-04","sale":694} +{"date":"2020-01-01","sale":193} +{"date":"2020-01-03","sale":531} +{"date":"2020-01-04","sale":291} +{"date":"2020-01-05","sale":683} +{"date":"2020-01-06","sale":774} +{"date":"2020-01-01","sale":844} +{"date":"2020-01-01","sale":276} +{"date":"2020-01-06","sale":431} +{"date":"2020-01-05","sale":691} +{"date":"2020-01-01","sale":543} +{"date":"2020-01-04","sale":468} +{"date":"2020-01-05","sale":60} +{"date":"2020-01-05","sale":520} +{"date":"2020-01-04","sale":133} +{"date":"2020-01-03","sale":517} +{"date":"2020-01-02","sale":269} +{"date":"2020-01-04","sale":758} +{"date":"2020-01-03","sale":868} +{"date":"2020-01-07","sale":257} +{"date":"2020-01-02","sale":86} +{"date":"2020-01-04","sale":778} +{"date":"2020-01-07","sale":344} +{"date":"2020-01-01","sale":877} +{"date":"2020-01-07","sale":582} +{"date":"2020-01-06","sale":809} +{"date":"2020-01-02","sale":282} +{"date":"2020-01-04","sale":295} +{"date":"2020-01-04","sale":712} +{"date":"2020-01-02","sale":634} +{"date":"2020-01-02","sale":468} +{"date":"2020-01-07","sale":328} +{"date":"2020-01-04","sale":109} +{"date":"2020-01-03","sale":55} +{"date":"2020-01-05","sale":139} +{"date":"2020-01-03","sale":267} +{"date":"2020-01-06","sale":523} +{"date":"2020-01-01","sale":875} +{"date":"2020-01-04","sale":331} +{"date":"2020-01-02","sale":299} +{"date":"2020-01-01","sale":33} +{"date":"2020-01-03","sale":484} +{"date":"2020-01-04","sale":982} +{"date":"2020-01-06","sale":960} +{"date":"2020-01-01","sale":216} +{"date":"2020-01-01","sale":122} +{"date":"2020-01-01","sale":85} +{"date":"2020-01-02","sale":618} +{"date":"2020-01-05","sale":123} +{"date":"2020-01-02","sale":286} +{"date":"2020-01-01","sale":540} +{"date":"2020-01-07","sale":916} +{"date":"2020-01-05","sale":255} +{"date":"2020-01-07","sale":243} +{"date":"2020-01-04","sale":850} +{"date":"2020-01-03","sale":114} +{"date":"2020-01-01","sale":483} +{"date":"2020-01-06","sale":40} +{"date":"2020-01-05","sale":110} +{"date":"2020-01-06","sale":467} +{"date":"2020-01-05","sale":656} +{"date":"2020-01-04","sale":988} +{"date":"2020-01-05","sale":419} +{"date":"2020-01-03","sale":567} +{"date":"2020-01-04","sale":442} +{"date":"2020-01-07","sale":744} +{"date":"2020-01-04","sale":820} +{"date":"2020-01-04","sale":673} +{"date":"2020-01-03","sale":77} +{"date":"2020-01-05","sale":368} +{"date":"2020-01-06","sale":96} +{"date":"2020-01-02","sale":293} +{"date":"2020-01-01","sale":817} +{"date":"2020-01-04","sale":682} +{"date":"2020-01-01","sale":246} +{"date":"2020-01-05","sale":979} +{"date":"2020-01-06","sale":870} +{"date":"2020-01-03","sale":156} +{"date":"2020-01-02","sale":954} +{"date":"2020-01-07","sale":306} +{"date":"2020-01-03","sale":936} +{"date":"2020-01-05","sale":881} +{"date":"2020-01-05","sale":118} +{"date":"2020-01-02","sale":181} +{"date":"2020-01-06","sale":979} +{"date":"2020-01-04","sale":541} +{"date":"2020-01-07","sale":33} +{"date":"2020-01-06","sale":41} +{"date":"2020-01-01","sale":7} +{"date":"2020-01-03","sale":684} +{"date":"2020-01-07","sale":91} +{"date":"2020-01-05","sale":484} +{"date":"2020-01-07","sale":517} +{"date":"2020-01-06","sale":958} +{"date":"2020-01-03","sale":846} +{"date":"2020-01-02","sale":999} +{"date":"2020-01-04","sale":367} +{"date":"2020-01-04","sale":742} +{"date":"2020-01-05","sale":62} +{"date":"2020-01-07","sale":669} +{"date":"2020-01-02","sale":843} +{"date":"2020-01-03","sale":733} +{"date":"2020-01-05","sale":409} +{"date":"2020-01-05","sale":649} +{"date":"2020-01-01","sale":449} +{"date":"2020-01-01","sale":59} +{"date":"2020-01-06","sale":864} +{"date":"2020-01-05","sale":755} +{"date":"2020-01-01","sale":40} +{"date":"2020-01-02","sale":366} +{"date":"2020-01-06","sale":804} +{"date":"2020-01-01","sale":875} +{"date":"2020-01-07","sale":788} +{"date":"2020-01-07","sale":481} +{"date":"2020-01-05","sale":465} +{"date":"2020-01-05","sale":988} +{"date":"2020-01-05","sale":10} +{"date":"2020-01-03","sale":758} +{"date":"2020-01-02","sale":493} +{"date":"2020-01-04","sale":738} +{"date":"2020-01-03","sale":330} +{"date":"2020-01-06","sale":500} +{"date":"2020-01-04","sale":345} +{"date":"2020-01-06","sale":740} +{"date":"2020-01-06","sale":932} +{"date":"2020-01-06","sale":17} +{"date":"2020-01-04","sale":471} +{"date":"2020-01-07","sale":890} +{"date":"2020-01-02","sale":507} +{"date":"2020-01-06","sale":895} +{"date":"2020-01-02","sale":274} +{"date":"2020-01-01","sale":927} +{"date":"2020-01-04","sale":756} +{"date":"2020-01-05","sale":795} +{"date":"2020-01-06","sale":504} +{"date":"2020-01-04","sale":223} +{"date":"2020-01-07","sale":839} +{"date":"2020-01-05","sale":17} +{"date":"2020-01-06","sale":11} +{"date":"2020-01-04","sale":138} +{"date":"2020-01-07","sale":773} +{"date":"2020-01-02","sale":569} +{"date":"2020-01-04","sale":435} +{"date":"2020-01-06","sale":47} +{"date":"2020-01-07","sale":287} +{"date":"2020-01-04","sale":573} +{"date":"2020-01-03","sale":829} +{"date":"2020-01-05","sale":745} +{"date":"2020-01-01","sale":701} +{"date":"2020-01-07","sale":800} +{"date":"2020-01-07","sale":230} +{"date":"2020-01-05","sale":373} +{"date":"2020-01-05","sale":812} +{"date":"2020-01-01","sale":103} +{"date":"2020-01-04","sale":344} +{"date":"2020-01-02","sale":113} +{"date":"2020-01-03","sale":334} +{"date":"2020-01-02","sale":451} +{"date":"2020-01-06","sale":668} +{"date":"2020-01-06","sale":590} +{"date":"2020-01-01","sale":958} +{"date":"2020-01-06","sale":813} +{"date":"2020-01-03","sale":22} +{"date":"2020-01-02","sale":39} +{"date":"2020-01-03","sale":8} +{"date":"2020-01-02","sale":392} +{"date":"2020-01-04","sale":116} +{"date":"2020-01-04","sale":775} +{"date":"2020-01-07","sale":221} +{"date":"2020-01-02","sale":587} +{"date":"2020-01-06","sale":358} +{"date":"2020-01-05","sale":246} +{"date":"2020-01-01","sale":812} +{"date":"2020-01-02","sale":108} +{"date":"2020-01-06","sale":479} +{"date":"2020-01-05","sale":20} +{"date":"2020-01-03","sale":66} +{"date":"2020-01-05","sale":285} +{"date":"2020-01-07","sale":2} +{"date":"2020-01-05","sale":716} +{"date":"2020-01-04","sale":811} +{"date":"2020-01-02","sale":161} +{"date":"2020-01-06","sale":701} +{"date":"2020-01-05","sale":858} +{"date":"2020-01-03","sale":286} +{"date":"2020-01-01","sale":951} +{"date":"2020-01-04","sale":697} +{"date":"2020-01-01","sale":269} +{"date":"2020-01-01","sale":364} +{"date":"2020-01-02","sale":518} +{"date":"2020-01-01","sale":135} +{"date":"2020-01-05","sale":813} +{"date":"2020-01-06","sale":240} +{"date":"2020-01-01","sale":815} +{"date":"2020-01-01","sale":892} +{"date":"2020-01-02","sale":765} +{"date":"2020-01-07","sale":368} +{"date":"2020-01-07","sale":262} +{"date":"2020-01-07","sale":885} +{"date":"2020-01-02","sale":266} +{"date":"2020-01-06","sale":298} +{"date":"2020-01-02","sale":465} +{"date":"2020-01-07","sale":723} +{"date":"2020-01-03","sale":985} +{"date":"2020-01-04","sale":823} +{"date":"2020-01-05","sale":518} +{"date":"2020-01-02","sale":464} +{"date":"2020-01-03","sale":316} +{"date":"2020-01-07","sale":484} +{"date":"2020-01-03","sale":849} +{"date":"2020-01-02","sale":464} +{"date":"2020-01-07","sale":670} +{"date":"2020-01-02","sale":747} +{"date":"2020-01-02","sale":675} +{"date":"2020-01-02","sale":728} +{"date":"2020-01-04","sale":546} +{"date":"2020-01-03","sale":606} +{"date":"2020-01-07","sale":11} +{"date":"2020-01-02","sale":66} +{"date":"2020-01-07","sale":31} +{"date":"2020-01-07","sale":492} +{"date":"2020-01-03","sale":209} +{"date":"2020-01-01","sale":183} +{"date":"2020-01-01","sale":814} +{"date":"2020-01-04","sale":49} +{"date":"2020-01-02","sale":981} +{"date":"2020-01-02","sale":546} +{"date":"2020-01-06","sale":26} +{"date":"2020-01-02","sale":58} +{"date":"2020-01-02","sale":348} +{"date":"2020-01-06","sale":779} +{"date":"2020-01-06","sale":822} +{"date":"2020-01-04","sale":958} +{"date":"2020-01-05","sale":491} +{"date":"2020-01-07","sale":385} +{"date":"2020-01-05","sale":466} +{"date":"2020-01-05","sale":878} +{"date":"2020-01-05","sale":49} +{"date":"2020-01-05","sale":327} +{"date":"2020-01-05","sale":654} +{"date":"2020-01-04","sale":132} +{"date":"2020-01-02","sale":753} +{"date":"2020-01-03","sale":663} +{"date":"2020-01-04","sale":917} +{"date":"2020-01-01","sale":143} +{"date":"2020-01-04","sale":165} +{"date":"2020-01-05","sale":513} +{"date":"2020-01-05","sale":299} +{"date":"2020-01-01","sale":822} +{"date":"2020-01-01","sale":811} +{"date":"2020-01-02","sale":597} +{"date":"2020-01-04","sale":559} +{"date":"2020-01-06","sale":376} +{"date":"2020-01-03","sale":692} +{"date":"2020-01-03","sale":876} +{"date":"2020-01-04","sale":841} +{"date":"2020-01-06","sale":386} +{"date":"2020-01-07","sale":279} +{"date":"2020-01-04","sale":650} +{"date":"2020-01-06","sale":212} +{"date":"2020-01-04","sale":664} +{"date":"2020-01-03","sale":98} +{"date":"2020-01-07","sale":614} +{"date":"2020-01-07","sale":805} +{"date":"2020-01-05","sale":545} +{"date":"2020-01-02","sale":857} +{"date":"2020-01-06","sale":24} +{"date":"2020-01-04","sale":697} +{"date":"2020-01-02","sale":151} +{"date":"2020-01-01","sale":908} +{"date":"2020-01-04","sale":242} +{"date":"2020-01-05","sale":210} +{"date":"2020-01-03","sale":772} +{"date":"2020-01-04","sale":29} +{"date":"2020-01-01","sale":744} +{"date":"2020-01-06","sale":195} +{"date":"2020-01-04","sale":918} +{"date":"2020-01-01","sale":15} +{"date":"2020-01-03","sale":183} +{"date":"2020-01-03","sale":831} +{"date":"2020-01-01","sale":956} +{"date":"2020-01-07","sale":696} +{"date":"2020-01-07","sale":248} +{"date":"2020-01-05","sale":549} +{"date":"2020-01-07","sale":655} +{"date":"2020-01-07","sale":551} +{"date":"2020-01-03","sale":670} +{"date":"2020-01-06","sale":142} +{"date":"2020-01-02","sale":777} +{"date":"2020-01-03","sale":550} +{"date":"2020-01-05","sale":515} +{"date":"2020-01-03","sale":709} +{"date":"2020-01-03","sale":480} +{"date":"2020-01-03","sale":896} +{"date":"2020-01-07","sale":599} +{"date":"2020-01-05","sale":653} +{"date":"2020-01-01","sale":43} +{"date":"2020-01-03","sale":412} +{"date":"2020-01-05","sale":401} +{"date":"2020-01-06","sale":238} +{"date":"2020-01-06","sale":291} +{"date":"2020-01-03","sale":311} +{"date":"2020-01-03","sale":75} +{"date":"2020-01-05","sale":729} +{"date":"2020-01-05","sale":11} +{"date":"2020-01-03","sale":637} +{"date":"2020-01-06","sale":649} +{"date":"2020-01-03","sale":934} +{"date":"2020-01-03","sale":737} +{"date":"2020-01-02","sale":945} +{"date":"2020-01-06","sale":682} +{"date":"2020-01-07","sale":132} +{"date":"2020-01-02","sale":594} +{"date":"2020-01-02","sale":640} +{"date":"2020-01-07","sale":341} +{"date":"2020-01-04","sale":585} +{"date":"2020-01-05","sale":425} +{"date":"2020-01-03","sale":698} +{"date":"2020-01-05","sale":149} +{"date":"2020-01-06","sale":477} +{"date":"2020-01-05","sale":528} +{"date":"2020-01-06","sale":104} +{"date":"2020-01-02","sale":197} +{"date":"2020-01-05","sale":380} +{"date":"2020-01-01","sale":188} +{"date":"2020-01-05","sale":998} +{"date":"2020-01-01","sale":788} +{"date":"2020-01-02","sale":480} +{"date":"2020-01-06","sale":581} +{"date":"2020-01-01","sale":394} +{"date":"2020-01-03","sale":874} +{"date":"2020-01-05","sale":306} +{"date":"2020-01-01","sale":587} +{"date":"2020-01-03","sale":91} +{"date":"2020-01-07","sale":680} +{"date":"2020-01-02","sale":305} +{"date":"2020-01-02","sale":418} +{"date":"2020-01-02","sale":578} +{"date":"2020-01-04","sale":916} +{"date":"2020-01-06","sale":856} +{"date":"2020-01-05","sale":894} +{"date":"2020-01-07","sale":404} +{"date":"2020-01-04","sale":682} +{"date":"2020-01-07","sale":636} +{"date":"2020-01-04","sale":839} +{"date":"2020-01-06","sale":37} +{"date":"2020-01-07","sale":771} +{"date":"2020-01-01","sale":868} +{"date":"2020-01-03","sale":566} +{"date":"2020-01-03","sale":437} +{"date":"2020-01-05","sale":657} +{"date":"2020-01-02","sale":823} +{"date":"2020-01-04","sale":877} +{"date":"2020-01-07","sale":127} +{"date":"2020-01-07","sale":617} +{"date":"2020-01-07","sale":264} +{"date":"2020-01-03","sale":491} +{"date":"2020-01-04","sale":325} +{"date":"2020-01-03","sale":399} +{"date":"2020-01-05","sale":333} +{"date":"2020-01-03","sale":381} +{"date":"2020-01-03","sale":213} +{"date":"2020-01-01","sale":261} +{"date":"2020-01-01","sale":659} +{"date":"2020-01-05","sale":509} +{"date":"2020-01-06","sale":734} +{"date":"2020-01-04","sale":179} +{"date":"2020-01-05","sale":619} +{"date":"2020-01-03","sale":319} +{"date":"2020-01-07","sale":747} +{"date":"2020-01-01","sale":64} +{"date":"2020-01-02","sale":120} +{"date":"2020-01-07","sale":950} +{"date":"2020-01-05","sale":278} +{"date":"2020-01-05","sale":351} +{"date":"2020-01-07","sale":934} +{"date":"2020-01-07","sale":483} +{"date":"2020-01-05","sale":685} +{"date":"2020-01-03","sale":82} +{"date":"2020-01-03","sale":980} +{"date":"2020-01-06","sale":390} +{"date":"2020-01-05","sale":905} +{"date":"2020-01-07","sale":859} +{"date":"2020-01-04","sale":890} +{"date":"2020-01-07","sale":370} +{"date":"2020-01-01","sale":210} +{"date":"2020-01-05","sale":181} +{"date":"2020-01-01","sale":539} +{"date":"2020-01-03","sale":557} +{"date":"2020-01-06","sale":331} +{"date":"2020-01-04","sale":791} +{"date":"2020-01-06","sale":20} +{"date":"2020-01-02","sale":93} +{"date":"2020-01-02","sale":299} +{"date":"2020-01-07","sale":768} +{"date":"2020-01-07","sale":179} +{"date":"2020-01-05","sale":59} +{"date":"2020-01-04","sale":410} +{"date":"2020-01-01","sale":731} +{"date":"2020-01-04","sale":544} +{"date":"2020-01-03","sale":91} +{"date":"2020-01-04","sale":167} +{"date":"2020-01-05","sale":509} +{"date":"2020-01-01","sale":854} +{"date":"2020-01-01","sale":34} +{"date":"2020-01-07","sale":352} +{"date":"2020-01-02","sale":145} +{"date":"2020-01-05","sale":114} +{"date":"2020-01-05","sale":30} +{"date":"2020-01-06","sale":458} +{"date":"2020-01-03","sale":210} +{"date":"2020-01-07","sale":203} +{"date":"2020-01-05","sale":441} +{"date":"2020-01-05","sale":308} +{"date":"2020-01-05","sale":137} +{"date":"2020-01-03","sale":64} +{"date":"2020-01-06","sale":134} +{"date":"2020-01-07","sale":224} +{"date":"2020-01-03","sale":970} +{"date":"2020-01-04","sale":445} +{"date":"2020-01-02","sale":482} +{"date":"2020-01-01","sale":570} +{"date":"2020-01-02","sale":604} +{"date":"2020-01-07","sale":845} +{"date":"2020-01-04","sale":324} +{"date":"2020-01-02","sale":822} +{"date":"2020-01-07","sale":437} +{"date":"2020-01-03","sale":224} +{"date":"2020-01-04","sale":428} +{"date":"2020-01-05","sale":181} +{"date":"2020-01-03","sale":337} +{"date":"2020-01-06","sale":607} +{"date":"2020-01-06","sale":895} +{"date":"2020-01-07","sale":784} +{"date":"2020-01-02","sale":427} +{"date":"2020-01-07","sale":962} +{"date":"2020-01-03","sale":113} +{"date":"2020-01-02","sale":251} +{"date":"2020-01-01","sale":105} +{"date":"2020-01-06","sale":137} +{"date":"2020-01-07","sale":689} +{"date":"2020-01-04","sale":651} +{"date":"2020-01-05","sale":564} +{"date":"2020-01-06","sale":456} +{"date":"2020-01-06","sale":616} +{"date":"2020-01-01","sale":136} +{"date":"2020-01-03","sale":685} +{"date":"2020-01-05","sale":96} +{"date":"2020-01-07","sale":481} +{"date":"2020-01-01","sale":571} +{"date":"2020-01-05","sale":353} +{"date":"2020-01-05","sale":308} +{"date":"2020-01-01","sale":906} +{"date":"2020-01-03","sale":737} +{"date":"2020-01-06","sale":510} +{"date":"2020-01-04","sale":961} +{"date":"2020-01-03","sale":737} +{"date":"2020-01-02","sale":661} +{"date":"2020-01-05","sale":195} +{"date":"2020-01-07","sale":796} +{"date":"2020-01-07","sale":928} +{"date":"2020-01-06","sale":209} +{"date":"2020-01-03","sale":139} +{"date":"2020-01-01","sale":311} +{"date":"2020-01-06","sale":969} +{"date":"2020-01-06","sale":813} +{"date":"2020-01-01","sale":250} +{"date":"2020-01-07","sale":589} +{"date":"2020-01-02","sale":10} +{"date":"2020-01-04","sale":366} +{"date":"2020-01-07","sale":357} +{"date":"2020-01-01","sale":105} +{"date":"2020-01-03","sale":682} +{"date":"2020-01-04","sale":584} +{"date":"2020-01-03","sale":758} +{"date":"2020-01-03","sale":970} +{"date":"2020-01-06","sale":545} +{"date":"2020-01-02","sale":36} +{"date":"2020-01-06","sale":374} +{"date":"2020-01-07","sale":810} +{"date":"2020-01-05","sale":11} +{"date":"2020-01-04","sale":98} +{"date":"2020-01-04","sale":233} +{"date":"2020-01-05","sale":705} +{"date":"2020-01-02","sale":653} +{"date":"2020-01-03","sale":111} +{"date":"2020-01-03","sale":341} +{"date":"2020-01-03","sale":926} +{"date":"2020-01-04","sale":118} +{"date":"2020-01-06","sale":592} +{"date":"2020-01-06","sale":530} +{"date":"2020-01-07","sale":606} +{"date":"2020-01-02","sale":945} diff --git a/sagemaker-core/tests/integ/processing/test_framework_processor_requirements.py b/sagemaker-core/tests/integ/processing/test_framework_processor_requirements.py new file mode 100644 index 0000000000..f98bb5b7a2 --- /dev/null +++ b/sagemaker-core/tests/integ/processing/test_framework_processor_requirements.py @@ -0,0 +1,169 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""End-to-end integration test locking in issue #5805. + +#5805: ``FrameworkProcessor.run(requirements=...)`` generated a ``runproc.sh`` +that hard-coded ``requirements.txt`` and never threaded the caller's +requirements file name through, so a differently named requirements file (or +none) was silently ignored and dependencies were not installed in the container. + +Two assertions, one cheap and one full: + +1. Cheap, job-outcome-independent (the exact regression): download the generated + ``runproc.sh`` from the job's uploaded ``entrypoint`` code location and assert + it references the requirements file name that was passed to ``run``. +2. Full: run the job to completion with an entry script that ``import art`` (a + small pure-python package NOT baked into the sklearn image) and writes a + marker file; a Completed status proves ``pip install -r `` ran + inside the container. +""" + +from __future__ import absolute_import + +import os +import tempfile +import time +import uuid + +import boto3 +import pytest + +from sagemaker.core import image_uris +from sagemaker.core.helper.session_helper import Session +from sagemaker.core.processing import FrameworkProcessor + +ROLE = "SageMakerRole" +REGION = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-west-2")) +INSTANCE_TYPE = "ml.m5.xlarge" +REQUIREMENTS_FILE_NAME = "requirements.txt" +# Small, pure-python, and NOT present in the sklearn processing image, so a +# Completed job proves the requirements install actually ran. +EXTRA_PACKAGE = "art==6.4" +WAIT_TIMEOUT_SECONDS = 30 * 60 +POLL_SECONDS = 30 +TERMINAL = ("Completed", "Failed", "Stopped") + +ENTRY_SCRIPT = """\ +import os + +import art # noqa: F401 -- from requirements.txt, absent in the base image + +out_dir = "/opt/ml/processing/output" +os.makedirs(out_dir, exist_ok=True) +with open(os.path.join(out_dir, "marker.txt"), "w") as fh: + fh.write("art import succeeded\\n") +""" + + +def _stop_quietly(client, job_name): + try: + if ( + client.describe_processing_job(ProcessingJobName=job_name)["ProcessingJobStatus"] + in TERMINAL + ): + return + client.stop_processing_job(ProcessingJobName=job_name) + except Exception: # pylint: disable=broad-except + pass + + +def _processing_image(): + return image_uris.retrieve( + "sklearn", REGION, version="1.2-1", py_version="py3", instance_type=INSTANCE_TYPE + ) + + +def _entrypoint_s3_uri(described): + """FrameworkProcessor uploads the generated runproc.sh as a ProcessingInput + named ``entrypoint``.""" + for inp in described.get("ProcessingInputs", []): + if inp.get("InputName") == "entrypoint": + return inp["S3Input"]["S3Uri"] + raise AssertionError(f"no 'entrypoint' ProcessingInput in Describe: {described}") + + +@pytest.mark.serial +def test_framework_processor_requirements_threaded_into_runproc_5805(): + """requirements= must reach runproc.sh and get installed in the container.""" + client = boto3.client("sagemaker", region_name=REGION) + session = Session(sagemaker_client=client) + s3_client = boto3.client("s3", region_name=REGION) + + processor = FrameworkProcessor( + image_uri=_processing_image(), + role=ROLE, + instance_count=1, + instance_type=INSTANCE_TYPE, + command=["python3"], + volume_size_in_gb=30, + max_runtime_in_seconds=1800, + sagemaker_session=session, + ) + + job_name = f"fw-proc-reqs-{uuid.uuid4().hex[:8]}" + source_dir = tempfile.mkdtemp() + entry = "process.py" + with open(os.path.join(source_dir, entry), "w") as fh: + fh.write(ENTRY_SCRIPT) + with open(os.path.join(source_dir, REQUIREMENTS_FILE_NAME), "w") as fh: + fh.write(EXTRA_PACKAGE + "\n") + + try: + processor.run( + code=entry, + source_dir=source_dir, + requirements=REQUIREMENTS_FILE_NAME, + wait=False, + logs=False, + job_name=job_name, + ) + + described = client.describe_processing_job(ProcessingJobName=job_name) + + # --- (1) The exact regression: runproc.sh must name the requirements file. + runproc_uri = _entrypoint_s3_uri(described) + bucket, key = runproc_uri[len("s3://") :].split("/", 1) + runproc_body = s3_client.get_object(Bucket=bucket, Key=key)["Body"].read().decode("utf-8") + assert REQUIREMENTS_FILE_NAME in runproc_body, ( + f"runproc.sh does not reference {REQUIREMENTS_FILE_NAME!r} -- requirements were " + f"not threaded through (#5805). runproc.sh:\n{runproc_body}" + ) + + # --- (2) Full proof: the install actually happened -> job Completed. + deadline = time.time() + WAIT_TIMEOUT_SECONDS + status = described["ProcessingJobStatus"] + while time.time() < deadline and status not in TERMINAL: + time.sleep(POLL_SECONDS) + status = client.describe_processing_job(ProcessingJobName=job_name)[ + "ProcessingJobStatus" + ] + + failure = client.describe_processing_job(ProcessingJobName=job_name).get( + "FailureReason", "" + ) + assert status == "Completed", ( + f"job status={status!r} (expected Completed); if the container could not " + f"`import art`, requirements were not installed (job={job_name}, " + f"failure={failure!r})" + ) + finally: + _stop_quietly(client, job_name) + for name in (entry, REQUIREMENTS_FILE_NAME): + try: + os.remove(os.path.join(source_dir, name)) + except OSError: + pass + try: + os.rmdir(source_dir) + except OSError: + pass diff --git a/sagemaker-core/tests/integ/processing/test_processor_code_kms_fallback.py b/sagemaker-core/tests/integ/processing/test_processor_code_kms_fallback.py new file mode 100644 index 0000000000..68f93c5ad3 --- /dev/null +++ b/sagemaker-core/tests/integ/processing/test_processor_code_kms_fallback.py @@ -0,0 +1,129 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""End-to-end integration test locking in issue #4874. + +#4874: ``ScriptProcessor.run(code=...)`` uploaded the user code file with no +server-side encryption even when the processor was configured with an +``output_kms_key``. The fix makes ``kms_key`` fall back to the processor's +``output_kms_key`` when the caller passes no explicit ``kms_key``. + +A pass proves the *upload path*, not the job outcome: we submit a real +processing job with ``output_kms_key`` set and NO ``kms_key`` argument, then +read the uploaded code object's S3 metadata straight from Describe and assert it +was encrypted with that same KMS key. The job is stopped immediately -- the fix +lives entirely in the request/upload path, so we never wait for completion. +""" + +from __future__ import absolute_import + +import os +import tempfile +import uuid + +import boto3 + +from sagemaker.core import image_uris +from sagemaker.core.helper.session_helper import Session +from sagemaker.core.processing import ScriptProcessor +from tests.integ.integ_test_kms_helpers import get_or_create_kms_key + +ROLE = "SageMakerRole" +REGION = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-west-2")) +INSTANCE_TYPE = "ml.m5.xlarge" +TERMINAL = ("Completed", "Failed", "Stopped") + + +def _stop_quietly(client, job_name): + """The upload has already happened by the time Describe returns the code + input, so there is no reason to let the job run on shared quota.""" + try: + if ( + client.describe_processing_job(ProcessingJobName=job_name)["ProcessingJobStatus"] + in TERMINAL + ): + return + client.stop_processing_job(ProcessingJobName=job_name) + except Exception: # pylint: disable=broad-except + pass + + +def _processing_image(): + return image_uris.retrieve( + "sklearn", REGION, version="1.2-1", py_version="py3", instance_type=INSTANCE_TYPE + ) + + +def _code_s3_uri(described): + """Pull the uploaded code object's S3 URI out of the Describe response. + + ScriptProcessor uploads the user code as a ProcessingInput named ``code``. + """ + for inp in described.get("ProcessingInputs", []): + if inp.get("InputName") == "code": + return inp["S3Input"]["S3Uri"] + raise AssertionError(f"no 'code' ProcessingInput in Describe: {described}") + + +def test_script_processor_code_kms_falls_back_to_output_kms_key_4874(): + """output_kms_key must encrypt the uploaded code when no kms_key is given.""" + client = boto3.client("sagemaker", region_name=REGION) + session = Session(sagemaker_client=client) + + role_arn = boto3.client("iam").get_role(RoleName=ROLE)["Role"]["Arn"] + kms_key_arn = get_or_create_kms_key(session, role_arn=role_arn) + key_id = kms_key_arn.split("/")[-1] + + processor = ScriptProcessor( + role=ROLE, + image_uri=_processing_image(), + command=["python3"], + instance_count=1, + instance_type=INSTANCE_TYPE, + volume_size_in_gb=30, + max_runtime_in_seconds=1800, + output_kms_key=kms_key_arn, + sagemaker_session=session, + ) + + job_name = f"code-kms-fallback-{uuid.uuid4().hex[:8]}" + tmpdir = tempfile.mkdtemp() + code_path = os.path.join(tmpdir, "noop.py") + with open(code_path, "w") as fh: + fh.write("print('hello from kms fallback test')\n") + + try: + # NOTE: no kms_key argument -- the fix must fall back to output_kms_key. + processor.run(code=code_path, wait=False, logs=False, job_name=job_name) + + described = client.describe_processing_job(ProcessingJobName=job_name) + code_uri = _code_s3_uri(described) + + bucket, key = code_uri[len("s3://") :].split("/", 1) + head = boto3.client("s3", region_name=REGION).head_object(Bucket=bucket, Key=key) + + assert head.get("ServerSideEncryption") == "aws:kms", ( + f"uploaded code was not KMS-encrypted (SSE={head.get('ServerSideEncryption')!r}); " + f"output_kms_key did not flow to the code upload (job={job_name})" + ) + sse_key = head.get("SSEKMSKeyId", "") + assert sse_key.endswith(key_id) or sse_key == kms_key_arn, ( + f"code encrypted with {sse_key!r}, expected the processor's output_kms_key " + f"{kms_key_arn!r} (job={job_name})" + ) + finally: + _stop_quietly(client, job_name) + try: + os.remove(code_path) + os.rmdir(tmpdir) + except OSError: + pass diff --git a/sagemaker-core/tests/integ/spark/__init__.py b/sagemaker-core/tests/integ/spark/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sagemaker-core/tests/integ/spark/test_spark_processing.py b/sagemaker-core/tests/integ/spark/test_spark_processing.py new file mode 100644 index 0000000000..9e77a232e7 --- /dev/null +++ b/sagemaker-core/tests/integ/spark/test_spark_processing.py @@ -0,0 +1,157 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""End-to-end integration tests for the V3 ``PySparkProcessor``. + +V3 shipped with NO Spark processing integ tests; this one is ported from V2's +``tests/integ/test_spark_processing.py`` to the V3 APIs and locks in two fixes +that are only observable against the real service: + +- #6253: the Spark event-log output stopped landing in S3. The job below + asserts the event-log S3 prefix is non-empty after the run. +- #6252: conf / py-files input channels were emitted in the wrong shape. We + assert their Describe ProcessingInputs carry an ``S3Input`` with a + ``LocalPath`` under ``/opt/ml/processing/input/``. + +#3809 (``submit_py_files`` must be a list) is an argument guard that fires before +any API call and is covered by unit tests. +""" + +from __future__ import absolute_import + +import os +import time +import uuid + +import boto3 +import pytest + +from sagemaker.core.helper.session_helper import Session +from sagemaker.core.spark.processing import PySparkProcessor + +ROLE = "SageMakerRole" +REGION = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-west-2")) +INSTANCE_TYPE = "ml.m5.xlarge" +FRAMEWORK_VERSION = "3.3" +WAIT_TIMEOUT_SECONDS = 30 * 60 +POLL_SECONDS = 20 +TERMINAL = ("Completed", "Failed", "Stopped") + +_DATA_DIR = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "spark") +_SPARK_APP = os.path.join(_DATA_DIR, "code", "python", "hello_py_spark", "hello_py_spark_app.py") +_SPARK_UDFS = os.path.join(_DATA_DIR, "code", "python", "hello_py_spark", "hello_py_spark_udfs.py") +_DATA_JSONL = os.path.join(_DATA_DIR, "files", "data.jsonl") + +CONFIGURATION = [ + { + "Classification": "spark-defaults", + "Properties": {"spark.executor.memory": "2g", "spark.executor.cores": "1"}, + }, +] + + +def _stop_quietly(client, job_name): + try: + if ( + client.describe_processing_job(ProcessingJobName=job_name)["ProcessingJobStatus"] + in TERMINAL + ): + return + client.stop_processing_job(ProcessingJobName=job_name) + except Exception: # pylint: disable=broad-except + pass + + +def _input_by_name(described, name): + for inp in described.get("ProcessingInputs", []): + if inp.get("InputName") == name: + return inp + return None + + +@pytest.mark.serial +def test_pyspark_multinode_event_logs_and_input_shape_6253_6252(): + """#6253 (event logs land in S3) and #6252 (conf/py-files input shape).""" + client = boto3.client("sagemaker", region_name=REGION) + session = Session(sagemaker_client=client) + s3_client = boto3.client("s3", region_name=REGION) + + processor = PySparkProcessor( + role=ROLE, + instance_count=2, + instance_type=INSTANCE_TYPE, + framework_version=FRAMEWORK_VERSION, + max_runtime_in_seconds=1800, + sagemaker_session=session, + ) + + bucket = session.default_bucket() + run_id = uuid.uuid4().hex[:8] + output_data_uri = f"s3://{bucket}/spark/output/sales/{run_id}" + event_logs_prefix = f"spark/spark-events/{run_id}" + event_logs_s3_uri = f"s3://{bucket}/{event_logs_prefix}" + + # Upload the input data set. + input_data_uri = f"s3://{bucket}/spark/input/{run_id}/data.jsonl" + with open(_DATA_JSONL) as data: + s3_client.put_object( + Bucket=bucket, Key=input_data_uri[len(f"s3://{bucket}/") :], Body=data.read() + ) + + job_name = f"pyspark-multinode-{run_id}" + try: + processor.run( + submit_app=_SPARK_APP, + submit_py_files=[_SPARK_UDFS], + arguments=["--input", input_data_uri, "--output", output_data_uri], + configuration=CONFIGURATION, + spark_event_logs_s3_uri=event_logs_s3_uri, + job_name=job_name, + wait=False, + ) + + described = client.describe_processing_job(ProcessingJobName=job_name) + + # --- #6252: conf and py-files channels carry the corrected S3Input shape. + for channel in ("conf", "py-files"): + inp = _input_by_name(described, channel) + assert inp is not None, f"no {channel!r} ProcessingInput: {described}" + assert "S3Input" in inp, f"{channel!r} input missing S3Input: {inp}" + local_path = inp["S3Input"].get("LocalPath", "") + assert local_path.startswith("/opt/ml/processing/input/"), ( + f"{channel!r} LocalPath={local_path!r} not under /opt/ml/processing/input/ " + f"(#6252 shape regression)" + ) + + # --- Wait for terminal, then #6253: event logs must be present in S3. + deadline = time.time() + WAIT_TIMEOUT_SECONDS + status = described["ProcessingJobStatus"] + while time.time() < deadline and status not in TERMINAL: + time.sleep(POLL_SECONDS) + status = client.describe_processing_job(ProcessingJobName=job_name)[ + "ProcessingJobStatus" + ] + + failure = client.describe_processing_job(ProcessingJobName=job_name).get( + "FailureReason", "" + ) + assert ( + status == "Completed" + ), f"job status={status!r} (expected Completed) job={job_name} failure={failure!r}" + + listed = s3_client.list_objects_v2(Bucket=bucket, Prefix=event_logs_prefix) + assert listed.get("KeyCount", 0) > 0, ( + f"no Spark event logs under s3://{bucket}/{event_logs_prefix} -- " + f"event-log output did not land in S3 (#6253)" + ) + finally: + _stop_quietly(client, job_name) diff --git a/sagemaker-core/tests/unit/spark/test_processing.py b/sagemaker-core/tests/unit/spark/test_processing.py new file mode 100644 index 0000000000..a79c782eee --- /dev/null +++ b/sagemaker-core/tests/unit/spark/test_processing.py @@ -0,0 +1,106 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from __future__ import absolute_import + +import pytest +from unittest.mock import Mock, patch + +from sagemaker.core.spark.processing import PySparkProcessor, _SparkProcessorBase +from sagemaker.core.shapes import ( + ProcessingInput, + ProcessingOutput, + ProcessingS3Input, + ProcessingS3Output, +) + + +@pytest.fixture +def mock_session(): + session = Mock() + session.boto_session = Mock() + session.boto_session.region_name = "us-west-2" + session.sagemaker_client = Mock() + session.default_bucket = Mock(return_value="test-bucket") + session.default_bucket_prefix = "sagemaker" + session.expand_role = Mock(side_effect=lambda x: x) + session.sagemaker_config = {} + return session + + +@pytest.fixture +def pyspark_processor(mock_session): + return PySparkProcessor( + role="arn:aws:iam::123456789012:role/SageMakerRole", + image_uri="test-spark-image:latest", + instance_count=1, + instance_type="ml.m5.xlarge", + sagemaker_session=mock_session, + ) + + +class TestStageSubmitDepsTypeValidation: + """#3809: a non-list submit dep must raise a clear ValueError, not iterate a string.""" + + def test_string_submit_deps_raises_clear_error(self, pyspark_processor): + with pytest.raises(ValueError, match="submit_deps must be a list"): + pyspark_processor._stage_submit_deps("s3://bucket/my.py", "py-files") + + def test_list_submit_deps_does_not_raise_type_error(self, pyspark_processor): + # A list of S3 URIs should be accepted and produce no input channel. + input_channel, spark_opt = pyspark_processor._stage_submit_deps( + ["s3://bucket/my.py"], "py-files" + ) + assert input_channel is None + assert spark_opt == "s3://bucket/my.py" + + +class TestSparkEventLogsOutputV3Shape: + """#6253: spark_event_logs_s3_uri output must use the V3 ProcessingOutput shape.""" + + def test_extend_processing_args_builds_v3_output(self, pyspark_processor): + _, outputs = pyspark_processor._extend_processing_args( + [], [], spark_event_logs_s3_uri="s3://bucket/spark-events/" + ) + assert outputs is not None and len(outputs) == 1 + output = outputs[0] + assert isinstance(output, ProcessingOutput) + assert output.output_name == "spark-event-logs" + assert isinstance(output.s3_output, ProcessingS3Output) + assert output.s3_output.s3_uri == "s3://bucket/spark-events/" + assert ( + output.s3_output.local_path == _SparkProcessorBase._spark_event_log_default_local_path + ) + assert output.s3_output.s3_upload_mode == "Continuous" + + +class TestStageSubmitDepsInputV3Shape: + """#6252: a local submit dep must build the V3 ProcessingInput shape.""" + + def test_local_dep_builds_v3_input(self, pyspark_processor): + with ( + patch("sagemaker.core.spark.processing.os.path.isfile", return_value=True), + patch("sagemaker.core.spark.processing.shutil.copy"), + patch("sagemaker.core.spark.processing.os.listdir", return_value=["my.py"]), + patch("sagemaker.core.spark.processing.S3Uploader.upload"), + ): + input_channel, spark_opt = pyspark_processor._stage_submit_deps( + ["/local/path/my.py"], "py-files" + ) + + assert isinstance(input_channel, ProcessingInput) + assert input_channel.input_name == "py-files" + assert isinstance(input_channel.s3_input, ProcessingS3Input) + assert input_channel.s3_input.s3_data_type == "S3Prefix" + assert input_channel.s3_input.s3_input_mode == "File" + # The spark-submit option points at the container-local mount path. + assert input_channel.s3_input.local_path in spark_opt diff --git a/sagemaker-core/tests/unit/test_processing.py b/sagemaker-core/tests/unit/test_processing.py index 559ac4179c..d54d2c4e81 100644 --- a/sagemaker-core/tests/unit/test_processing.py +++ b/sagemaker-core/tests/unit/test_processing.py @@ -2292,3 +2292,122 @@ def test_sparkjar_processor_forwards_instance_preferences(self, mock_session): sagemaker_session=mock_session, ) assert processor.instance_preferences == self._PREFS + + +class TestFrameworkProcessorRequirements: + """#5805: FrameworkProcessor.run(requirements=...) must be honored in the runproc script.""" + + def _make_processor(self, mock_session): + return FrameworkProcessor( + image_uri="test-image:latest", + role="arn:aws:iam::123456789012:role/SageMakerRole", + instance_count=1, + instance_type="ml.m5.xlarge", + command=["python3"], + sagemaker_session=mock_session, + ) + + def test_custom_requirements_file_is_used(self, mock_session): + processor = self._make_processor(mock_session) + script = processor._generate_framework_script( + "run_entrypoint.py", requirements="cpu-requirements.txt" + ) + assert "if [[ -f 'cpu-requirements.txt' ]]; then" in script + assert "install_requirements.py cpu-requirements.txt" in script + assert "if [[ -f 'requirements.txt' ]]; then" not in script + + def test_requirements_relative_subdirectory_is_preserved(self, mock_session): + """requirements is relative to source_dir, whose layout the bundle preserves.""" + processor = self._make_processor(mock_session) + script = processor._generate_framework_script( + "run_entrypoint.py", requirements="reqs/cpu-requirements.txt" + ) + assert "if [[ -f 'reqs/cpu-requirements.txt' ]]; then" in script + assert "install_requirements.py reqs/cpu-requirements.txt" in script + + def test_requirements_absolute_path_uses_basename(self, mock_session): + processor = self._make_processor(mock_session) + script = processor._generate_framework_script( + "run_entrypoint.py", requirements="/home/me/proj/cpu-requirements.txt" + ) + assert "if [[ -f 'cpu-requirements.txt' ]]; then" in script + + def test_custom_requirements_file_used_with_entry_point_s3_source(self, mock_session): + """The custom-entrypoint (S3 source_dir) branch must honor requirements too.""" + processor = self._make_processor(mock_session) + script = processor._generate_framework_script( + "run_entrypoint.py", + entry_point="runproc.sh", + source_dir="s3://bucket/code/sourcedir.tar.gz", + requirements="cpu-requirements.txt", + ) + assert "if [[ -f 'cpu-requirements.txt' ]]; then" in script + assert "install_requirements.py cpu-requirements.txt" in script + assert "if [[ -f 'requirements.txt' ]]; then" not in script + assert "./runproc.sh" in script + + def test_default_requirements_file_when_none(self, mock_session): + processor = self._make_processor(mock_session) + script = processor._generate_framework_script("run_entrypoint.py") + assert "if [[ -f 'requirements.txt' ]]; then" in script + assert "install_requirements.py requirements.txt" in script + + +class TestProcessorKmsKeyDefault: + """#4874: Processor.run kms_key should default to the configured output_kms_key.""" + + def test_run_defaults_kms_key_to_output_kms_key(self, mock_session): + processor = Processor( + role="arn:aws:iam::123456789012:role/SageMakerRole", + image_uri="test-image:latest", + instance_count=1, + instance_type="ml.m5.xlarge", + output_kms_key="arn:aws:kms:us-west-2:123456789012:key/out-key", + sagemaker_session=mock_session, + ) + + captured = {} + + def fake_normalize(**kwargs): + captured["kms_key"] = kwargs.get("kms_key") + return [], [] + + with ( + patch.object(processor, "_normalize_args", side_effect=fake_normalize), + patch.object(processor, "_start_new", return_value=Mock()), + patch( + "sagemaker.core.processing.check_and_get_run_experiment_config", + side_effect=lambda x: x, + ), + ): + processor.run(wait=False, logs=False) + + assert captured["kms_key"] == "arn:aws:kms:us-west-2:123456789012:key/out-key" + + def test_run_explicit_kms_key_wins(self, mock_session): + processor = Processor( + role="arn:aws:iam::123456789012:role/SageMakerRole", + image_uri="test-image:latest", + instance_count=1, + instance_type="ml.m5.xlarge", + output_kms_key="arn:aws:kms:us-west-2:123456789012:key/out-key", + sagemaker_session=mock_session, + ) + + captured = {} + + def fake_normalize(**kwargs): + captured["kms_key"] = kwargs.get("kms_key") + return [], [] + + with ( + patch.object(processor, "_normalize_args", side_effect=fake_normalize), + patch.object(processor, "_start_new", return_value=Mock()), + patch( + "sagemaker.core.processing.check_and_get_run_experiment_config", + side_effect=lambda x: x, + ), + ): + processor.run(wait=False, logs=False, kms_key="arn:aws:kms:us-west-2:1:key/explicit") + + assert captured["kms_key"] == "arn:aws:kms:us-west-2:1:key/explicit"