Training profiler support (#77)

- Requeue for profiler jobs in training
- Refactor to reduce code duplication and use common pkg
- Includes some test refactoring to include both profiler and debugger
- Clean up code duplication in test_training and test_training_debugger

**TODO**
fix docs in crd fields getting changed

### Testing

```
(ack) ubuntu@ip-172-31-0-119:~/go/src/github.com/aws-controllers-k8s/sagemaker-controller/test/e2e$ pytest tests/test_trainingjob.py 
========================================== test session starts ==========================================
platform linux -- Python 3.8.8, pytest-6.2.3, py-1.10.0, pluggy-0.13.1
rootdir: /home/ubuntu/go/src/github.com/aws-controllers-k8s/sagemaker-controller/test/e2e
plugins: xdist-2.2.0, forked-1.3.0
collected 2 items                                                                                       

tests/test_trainingjob.py ..                                                                      [100%]

===================================== 2 passed in 441.19s (0:07:21) =====================================
```

```
(ack) ubuntu@ip-172-31-0-119:~/go/src/github.com/aws-controllers-k8s/sagemaker-controller/test/e2e$ pytest tests/test_trainingjob_debugger.py 
========================================== test session starts ==========================================
platform linux -- Python 3.8.8, pytest-6.2.3, py-1.10.0, pluggy-0.13.1
rootdir: /home/ubuntu/go/src/github.com/aws-controllers-k8s/sagemaker-controller/test/e2e
plugins: xdist-2.2.0, forked-1.3.0
collected 1 item                                                                                        

tests/test_trainingjob_debugger.py .                                                              [100%]

===================================== 1 passed in 629.97s (0:10:29) =====================================
```

Author: surajkota

apis/v1alpha1/ack-generate-metadata.yaml

@@ -1,14 +1,14 @@
 ack_generate_info:
-  build_date: "2021-08-12T00:32:37Z"
+  build_date: "2021-08-12T23:35:21Z"
   build_hash: c77aa9c75d944952dee198029ba9822691cd82b0
   go_version: go1.16.4 linux/amd64
   version: v0.6.0
-api_directory_checksum: c8f263a8d1a2e21f53c85b1ca5e2be3687045dd0
+api_directory_checksum: f7275158658c496010e6e3d2d24b3aba37e2e1f4
 api_version: v1alpha1
 aws_sdk_go_version: v1.38.11
 generator_config_info:
-  file_checksum: 795f6cf17e64a254ed063ee36e489b2949a031ef
+  file_checksum: 4acc645991dfd96fd543c3c34df274a8e4f4787e
   original_file_name: generator.yaml
 last_modification:
   reason: API generation
-  timestamp: 2021-08-12 00:32:40.304705695 +0000 UTC
+  timestamp: 2021-08-12 23:35:25.778426245 +0000 UTC

apis/v1alpha1/generator.yaml

@@ -1,6 +1,4 @@
 operations:
-  DescribeTrainingJob:
-    set_output_custom_method_name: customDescribeTrainingJobSetOutput
   StopTrainingJob:
     operation_type: Delete
     resource_name: TrainingJob
@@ -169,7 +167,9 @@ resources:
       delta_pre_compare:
         code: customSetDefaults(a, b)
       sdk_create_post_set_output:
-        code: rm.customSetOutput(desired, aws.String(svcsdk.TrainingJobStatusInProgress), ko)
+        code: rm.customCreateTrainingJobSetOutput(&resource{ko})
+      sdk_read_one_post_set_output:
+        code: rm.customDescribeTrainingJobSetOutput(&resource{ko})
       sdk_delete_pre_build_request:
         template_path: training_job/sdk_delete_pre_build_request.go.tpl
     fields:
@@ -191,7 +191,17 @@ resources:
         is_read_only: true
         from:
           operation: DescribeTrainingJob
-          path: DebugRuleEvaluationStatuses 
+          path: DebugRuleEvaluationStatuses
+      ProfilerRuleEvaluationStatuses:
+        is_read_only: true
+        from:
+          operation: DescribeTrainingJob
+          path: ProfilerRuleEvaluationStatuses
+      ModelArtifacts:
+        is_read_only: true
+        from:
+          operation: DescribeTrainingJob
+          path: ModelArtifacts
       FailureReason:
         is_read_only: true
         print:

apis/v1alpha1/training_job.go

@@ -516,6 +516,32 @@ spec:
               failureReason:
                 description: If the training job failed, the reason it failed.
                 type: string
+              modelArtifacts:
+                description: Information about the Amazon S3 location that is configured
+                  for storing model artifacts.
+                properties:
+                  s3ModelArtifacts:
+                    type: string
+                type: object
+              profilerRuleEvaluationStatuses:
+                description: Evaluation status of Debugger rules for profiling on
+                  a training job.
+                items:
+                  description: Information about the status of the rule evaluation.
+                  properties:
+                    lastModifiedTime:
+                      format: date-time
+                      type: string
+                    ruleConfigurationName:
+                      type: string
+                    ruleEvaluationJobARN:
+                      type: string
+                    ruleEvaluationStatus:
+                      type: string
+                    statusDetails:
+                      type: string
+                  type: object
+                type: array
               secondaryStatus:
                 description: "Provides detailed information about the state of the
                   training job. For detailed information on the secondary status of

apis/v1alpha1/zz_generated.deepcopy.go

@@ -1,6 +1,4 @@
 operations:
-  DescribeTrainingJob:
-    set_output_custom_method_name: customDescribeTrainingJobSetOutput
   StopTrainingJob:
     operation_type: Delete
     resource_name: TrainingJob
@@ -169,7 +167,9 @@ resources:
       delta_pre_compare:
         code: customSetDefaults(a, b)
       sdk_create_post_set_output:
-        code: rm.customSetOutput(desired, aws.String(svcsdk.TrainingJobStatusInProgress), ko)
+        code: rm.customCreateTrainingJobSetOutput(&resource{ko})
+      sdk_read_one_post_set_output:
+        code: rm.customDescribeTrainingJobSetOutput(&resource{ko})
       sdk_delete_pre_build_request:
         template_path: training_job/sdk_delete_pre_build_request.go.tpl
     fields:
@@ -191,7 +191,17 @@ resources:
         is_read_only: true
         from:
           operation: DescribeTrainingJob
-          path: DebugRuleEvaluationStatuses 
+          path: DebugRuleEvaluationStatuses
+      ProfilerRuleEvaluationStatuses:
+        is_read_only: true
+        from:
+          operation: DescribeTrainingJob
+          path: ProfilerRuleEvaluationStatuses
+      ModelArtifacts:
+        is_read_only: true
+        from:
+          operation: DescribeTrainingJob
+          path: ModelArtifacts
       FailureReason:
         is_read_only: true
         print:

config/crd/bases/sagemaker.services.k8s.aws_trainingjobs.yaml

@@ -17,75 +17,50 @@
 package training_job
 
 import (
-	"context"
-
-	ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1"
-	svcapitypes "github.com/aws-controllers-k8s/sagemaker-controller/apis/v1alpha1"
+	svccommon "github.com/aws-controllers-k8s/sagemaker-controller/pkg/common"
 	"github.com/aws/aws-sdk-go/aws"
 	svcsdk "github.com/aws/aws-sdk-go/service/sagemaker"
-	corev1 "k8s.io/api/core/v1"
 )
 
-// customDescribeTrainingJobSetOutput sets the resource ResourceSynced condition to False if
-// TrainingJob is being modified by AWS. It has an additional check on the debugger status.
-func (rm *resourceManager) customDescribeTrainingJobSetOutput(
-	ctx context.Context,
-	r *resource,
-	resp *svcsdk.DescribeTrainingJobOutput,
-	ko *svcapitypes.TrainingJob,
-) (*svcapitypes.TrainingJob, error) {
-	trainingJobStatus := resp.TrainingJobStatus
-	debuggerRuleInProgress := false
-	if resp.DebugRuleEvaluationStatuses != nil {
-		for _, rule := range resp.DebugRuleEvaluationStatuses {
-			if rule.RuleEvaluationStatus != nil && *rule.RuleEvaluationStatus == svcsdk.RuleEvaluationStatusInProgress {
-				debuggerRuleInProgress = true
-				rm.customSetOutput(r, aws.String(svcsdk.TrainingJobStatusInProgress), ko)
-				break
-			}
-		}
+var (
+	trainingJobModifyingStatuses = []string{
+		svcsdk.TrainingJobStatusInProgress,
+		svcsdk.TrainingJobStatusStopping,
 	}
-
-	if !debuggerRuleInProgress {
-		rm.customSetOutput(r, trainingJobStatus, ko)
+	ruleModifyingStatuses = []string{
+		svcsdk.RuleEvaluationStatusInProgress,
+		svcsdk.RuleEvaluationStatusStopping,
 	}
+	resourceName = resourceGK.Kind
+)
 
-	return ko, nil
-}
-
-// customSetOutput sets ConditionTypeResourceSynced condition to True or False
-// based on the trainingJobStatus on AWS so the reconciler can determine if a
-// requeue is needed
-func (rm *resourceManager) customSetOutput(
-	r *resource,
-	trainingJobStatus *string,
-	ko *svcapitypes.TrainingJob,
-) {
-	if trainingJobStatus == nil {
+// customDescribeTrainingJobSetOutput sets the resource ResourceSynced condition to False if
+// TrainingJob is being modified by AWS. It checks for debug and profiler rule status in addition to TrainingJobStatus
+func (rm *resourceManager) customDescribeTrainingJobSetOutput(r *resource) {
+	trainingJobStatus := r.ko.Status.TrainingJobStatus
+	// early exit if training job is InProgress
+	if trainingJobStatus != nil && *trainingJobStatus == svcsdk.TrainingJobStatusInProgress {
+		svccommon.SetSyncedCondition(r, trainingJobStatus, &resourceName, &trainingJobModifyingStatuses)
 		return
 	}
 
-	syncConditionStatus := corev1.ConditionUnknown
-	if *trainingJobStatus == svcsdk.TrainingJobStatusCompleted || *trainingJobStatus == svcsdk.TrainingJobStatusStopped || *trainingJobStatus == svcsdk.TrainingJobStatusFailed {
-		syncConditionStatus = corev1.ConditionTrue
-	} else {
-		syncConditionStatus = corev1.ConditionFalse
-	}
-
-	var resourceSyncedCondition *ackv1alpha1.Condition = nil
-	for _, condition := range ko.Status.Conditions {
-		if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
-			resourceSyncedCondition = condition
-			break
+	for _, rule := range r.ko.Status.DebugRuleEvaluationStatuses {
+		if rule.RuleEvaluationStatus != nil && svccommon.IsModifyingStatus(rule.RuleEvaluationStatus, &ruleModifyingStatuses) {
+			svccommon.SetSyncedCondition(r, rule.RuleEvaluationStatus, aws.String("DebugRule"), &ruleModifyingStatuses)
+			return
 		}
 	}
 
-	if resourceSyncedCondition == nil {
-		resourceSyncedCondition = &ackv1alpha1.Condition{
-			Type: ackv1alpha1.ConditionTypeResourceSynced,
+	for _, rule := range r.ko.Status.ProfilerRuleEvaluationStatuses {
+		if rule.RuleEvaluationStatus != nil && svccommon.IsModifyingStatus(rule.RuleEvaluationStatus, &ruleModifyingStatuses) {
+			svccommon.SetSyncedCondition(r, rule.RuleEvaluationStatus, aws.String("ProfilerRule"), &ruleModifyingStatuses)
+			return
 		}
-		ko.Status.Conditions = append(ko.Status.Conditions, resourceSyncedCondition)
 	}
-	resourceSyncedCondition.Status = syncConditionStatus
 
+	svccommon.SetSyncedCondition(r, trainingJobStatus, &resourceName, &trainingJobModifyingStatuses)
+}
+
+func (rm *resourceManager) customCreateTrainingJobSetOutput(r *resource) {
+	svccommon.SetSyncedCondition(r, aws.String(svcsdk.TrainingJobStatusInProgress), &resourceName, &trainingJobModifyingStatuses)
 }