Increasing test timeouts and disabling stop tests (#235)

Issue #, if available:

Description of changes:

Monitoring Schedule can sometimes reach the scheduled state before the controller can verify that it is in Pending state. This PR removes the check for pending/resource synced.

Also increasing timeouts and disabling test_*_stopped tests

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: ananth102

test/e2e/tests/test_feature_group.py

@@ -38,7 +38,7 @@
 FEATURE_GROUP_STATUS_CREATING = "Creating"
 FEATURE_GROUP_STATUS_CREATED = "Created"
 # longer wait is used because we sometimes see server taking time to create/delete
-WAIT_PERIOD_COUNT = 8
+WAIT_PERIOD_COUNT = 9
 WAIT_PERIOD_LENGTH = 30
 STATUS = "status"
 RESOURCE_STATUS = "featureGroupStatus"
@@ -144,12 +144,6 @@ def test_create_feature_group(self, feature_group):
 
         assert k8s.get_resource_arn(resource) == feature_group_arn
 
-        assert (
-            feature_group_sm_desc["FeatureGroupStatus"] == FEATURE_GROUP_STATUS_CREATING
-        )
-
-        assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
-
         self._assert_feature_group_status_in_sync(
             feature_group_name, reference, FEATURE_GROUP_STATUS_CREATED
         )

test/e2e/tests/test_hpo.py

@@ -86,7 +86,6 @@ def get_hpo_resource_status(reference: k8s.CustomResourceReference):
 
 
 @service_marker
-@pytest.mark.canary
 class TestHPO:
     def _wait_resource_hpo_status(
         self,
@@ -153,6 +152,7 @@ def test_stopped(self, xgboost_hpojob):
             hpo_sm_desc["HyperParameterTuningJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED
         )
 
+    @pytest.mark.canary
     def test_completed(self, xgboost_hpojob):
         (reference, resource) = xgboost_hpojob
         assert k8s.get_resource_exists(reference)

test/e2e/tests/test_model_package.py

@@ -132,7 +132,7 @@ def _wait_resource_model_package_status(
         self,
         reference: k8s.CustomResourceReference,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 32,
         period_length: int = 30,
     ):
         return wait_for_status(
@@ -147,7 +147,7 @@ def _wait_sagemaker_model_package_status(
         self,
         model_package_name,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 32,
         period_length: int = 30,
     ):
         return wait_for_status(

test/e2e/tests/test_monitoring_schedule.py

@@ -141,7 +141,7 @@ def _assert_monitoring_schedule_status_in_sync(
         schedule_name,
         reference,
         expected_status,
-        wait_periods: int = 6,
+        wait_periods: int = 7,
         period_length: int = 30,
     ):
         assert (
@@ -165,19 +165,6 @@ def test_smoke(
         monitoring_schedule_arn = monitoring_schedule_desc["MonitoringScheduleArn"]
         assert k8s.get_resource_arn(resource) == monitoring_schedule_arn
 
-        # scheule transitions Pending -> Scheduled state
-        # Pending status is shortlived only for 30 seconds because baselining job has already been run
-        # remove the checks for Pending status if the test is flaky because of this
-        # as the main objective is to test for Scheduled status
-        # OR
-        # create the schedule with a on-going baseline job where it waits for the baselining job to complete
-        assert (
-            wait_resource_monitoring_schedule_status(
-                reference, self.STATUS_PENDING, 5, 2
-            )
-            == self.STATUS_PENDING
-        )
-        assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False", 5, 2)
 
         self._assert_monitoring_schedule_status_in_sync(
             sagemaker_client, monitoring_schedule_name, reference, self.STATUS_SCHEDULED

test/e2e/tests/test_processingjob.py

@@ -86,7 +86,6 @@ def get_processing_resource_status(reference: k8s.CustomResourceReference):
 
 
 @service_marker
-@pytest.mark.canary
 class TestProcessingJob:
     def _wait_resource_processing_status(
         self,
@@ -152,6 +151,7 @@ def test_stopped(self, kmeans_processing_job):
         processing_job_desc = get_sagemaker_processing_job(processing_job_name)
         assert processing_job_desc["ProcessingJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED
 
+    @pytest.mark.canary
     def test_completed(self, kmeans_processing_job):
         (reference, resource) = kmeans_processing_job
         assert k8s.get_resource_exists(reference)

test/e2e/tests/test_trainingjob.py

@@ -58,7 +58,6 @@ def xgboost_training_job():
     )
 
 
-@pytest.mark.canary
 @service_marker
 class TestTrainingJob:
     def test_stopped(self, xgboost_training_job):
@@ -86,6 +85,7 @@ def test_stopped(self, xgboost_training_job):
         training_job_desc = get_sagemaker_training_job(training_job_name)
         assert training_job_desc["TrainingJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED
 
+    @pytest.mark.canary
     def test_completed(self, xgboost_training_job):
         (reference, resource) = xgboost_training_job
         assert k8s.get_resource_exists(reference)

test/e2e/tests/test_transformjob.py

@@ -123,13 +123,12 @@ def get_transform_resource_status(reference: k8s.CustomResourceReference):
 
 
 @service_marker
-@pytest.mark.canary
 class TestTransformJob:
     def _wait_resource_transform_status(
         self,
         reference: k8s.CustomResourceReference,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 32,
         period_length: int = 30,
     ):
         return wait_for_status(
@@ -144,7 +143,7 @@ def _wait_sagemaker_transform_status(
         self,
         transform_job_name,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 32,
         period_length: int = 30,
     ):
         return wait_for_status(
@@ -188,6 +187,7 @@ def test_stopped(self, xgboost_transformjob):
         transform_sm_desc = get_sagemaker_transform_job(transform_job_name)
         assert transform_sm_desc["TransformJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED
 
+    @pytest.mark.canary
     def test_completed(self, xgboost_transformjob):
         (reference, resource) = xgboost_transformjob
         assert k8s.get_resource_exists(reference)