Increase timeout for certain e2e status checks and re-order cleanup (#117)

Description of changes:

- Some Canary e2e tests are failing with status mismatch ex: `Inprogress != Completed`, even though the jobs will eventually pass.
- Reached out to service teams for investigation in what may be occurring on these jobs.  
- Some resources are order dependent for deletion ex: Cannot delete an `endpoint` if it has a `monitoringSchedule` applied to it.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: ryansteakley

test/canary/scripts/run_test.sh

@@ -30,6 +30,7 @@ function print_controller_logs() {
 function cleanup {
   echo "Cleaning up resources"
   set +e
+  kubectl delete monitoringschedules --all
   kubectl delete endpoints.sagemaker --all
   kubectl delete endpointconfigs --all
   kubectl delete models --all
@@ -41,11 +42,10 @@ function cleanup {
   kubectl delete modelbiasjobdefinitions --all
   kubectl delete modelexplainabilityjobdefinitions --all
   kubectl delete modelqualityjobdefinitions --all
-  kubectl delete monitoringschedules --all
   kubectl delete adoptedresources --all
   kubectl delete featuregroups --all
-  kubectl delete modelpackagegroups --all
   kubectl delete modelpackages --all
+  kubectl delete modelpackagegroups --all
   kubectl delete notebookinstances --all
   kubectl delete notebookinstancelifecycleconfig --all
 

test/e2e/tests/test_feature_group.py

@@ -37,7 +37,7 @@
 FEATURE_GROUP_STATUS_CREATING = "Creating"
 FEATURE_GROUP_STATUS_CREATED = "Created"
 # longer wait is used because we sometimes see server taking time to create/delete
-WAIT_PERIOD_COUNT = 4
+WAIT_PERIOD_COUNT = 8
 WAIT_PERIOD_LENGTH = 30
 STATUS = "status"
 RESOURCE_STATUS = "featureGroupStatus"

test/e2e/tests/test_hpo.py

@@ -91,7 +91,7 @@ def _wait_resource_hpo_status(
         self,
         reference: k8s.CustomResourceReference,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 45,
         period_length: int = 30,
     ):
         return wait_for_status(
@@ -106,7 +106,7 @@ def _wait_sagemaker_hpo_status(
         self,
         hpo_job_name,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 45,
         period_length: int = 30,
     ):
         return wait_for_status(