fix: Use server side defaults for Train definitions (#177)

Issue:
We [do not](https://github.com/aws-controllers-k8s/sagemaker-controller/blob/3996ba037349bb0b65088341afdfd03cf657e09c/generator.yaml#L326) late initialize some parameters in TrainingJobDefinitions like we do in TrainingJob defintion. As a result the controller will infinitely requeue if all parameters are not explicity specified(because the server sends back the default values it uses).

Description of changes:

`pkg/resource/hyper_parameter_tuning_job/custom_delta.go` - Sets some parameters to their server side default.
`pkg/resource/hyper_parameter_tuning_job/testdata/v1alpha1/readone/observed/completed_variation.yaml` - Modified unit test.


CRD I used to test:

```
apiVersion: sagemaker.services.k8s.aws/v1alpha1
kind: HyperParameterTuningJob
metadata:
  name: 2022-10-31-hpo-3
spec:
  hyperParameterTuningJobName: 2022-10-31-hpo-3
  hyperParameterTuningJobConfig:
    strategy: Bayesian
    resourceLimits:
      maxNumberOfTrainingJobs: 2
      maxParallelTrainingJobs: 1
    trainingJobEarlyStoppingType: Auto
  trainingJobDefinitions:
  - staticHyperParameters:
      base_score: '0.5'
    definitionName: training-job-for-hpo
    algorithmSpecification:
      trainingImage: 433757028032.dkr.ecr.us-west-2.amazonaws.com/xgboost:1
      trainingInputMode: File
    roleARN: <arn>
    tuningObjective:
      type_: Minimize
      metricName: validation:error
    hyperParameterRanges:
      integerParameterRanges:
      - name: num_round
        minValue: '10'
        maxValue: '20'
        scalingType: Linear
      continuousParameterRanges:
      - name: gamma
        minValue: '0'
        maxValue: '5'
        scalingType: Linear
    inputDataConfig:
    - channelName: train
      dataSource:
        s3DataSource:
          s3DataType: S3Prefix
          s3URI: <train>
          s3DataDistributionType: FullyReplicated
      contentType: text/libsvm
      compressionType: None
      recordWrapperType: None
      inputMode: File
    - channelName: validation
      dataSource:
        s3DataSource:
          s3DataType: S3Prefix
          s3URI: <validation>
          s3DataDistributionType: FullyReplicated
      contentType: text/libsvm
      compressionType: None
      recordWrapperType: None
      inputMode: File
    outputDataConfig:
      s3OutputPath: <output>
    resourceConfig:
      instanceType: ml.m5.large
      instanceCount: 1
      volumeSizeInGB: 25
    stoppingCondition:
      maxRuntimeInSeconds: 3600
    enableNetworkIsolation: true
    enableInterContainerTrafficEncryption: false
  tags:
    - key: algorithm
      value: xgboost
    - key: environment
      value: testing
    - key: customer
      value: test-user
```

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: ananth102

pkg/resource/hyper_parameter_tuning_job/custom_delta.go

@@ -32,4 +32,35 @@ func customSetDefaults(
 			}
 		}
 	}
+
+	// TODO: Use late initialize instead once code generator supports late initializing slices.
+	if ackcompare.IsNotNil(a.ko.Spec.TrainingJobDefinitions) && ackcompare.IsNotNil(b.ko.Spec.TrainingJobDefinitions) {
+		if len(a.ko.Spec.TrainingJobDefinitions) == len(b.ko.Spec.TrainingJobDefinitions) {
+			for index := range a.ko.Spec.TrainingJobDefinitions {
+				latestStaticHyperParameters := b.ko.Spec.TrainingJobDefinitions[index].StaticHyperParameters
+				if ackcompare.IsNotNil(latestStaticHyperParameters) {
+					for key, _ := range latestStaticHyperParameters {
+						if key[0:1] == "_" {
+							delete(b.ko.Spec.TrainingJobDefinitions[index].StaticHyperParameters, key)
+						}
+					}
+				}
+				if ackcompare.IsNotNil(a.ko.Spec.TrainingJobDefinitions[index].AlgorithmSpecification) && ackcompare.IsNotNil(b.ko.Spec.TrainingJobDefinitions[index].AlgorithmSpecification) {
+					if ackcompare.IsNil(a.ko.Spec.TrainingJobDefinitions[index].AlgorithmSpecification.MetricDefinitions) && ackcompare.IsNotNil(b.ko.Spec.TrainingJobDefinitions[index].AlgorithmSpecification.MetricDefinitions) {
+						a.ko.Spec.TrainingJobDefinitions[index].AlgorithmSpecification.MetricDefinitions = b.ko.Spec.TrainingJobDefinitions[index].AlgorithmSpecification.MetricDefinitions
+					}
+				}
+				if ackcompare.IsNil(a.ko.Spec.TrainingJobDefinitions[index].EnableInterContainerTrafficEncryption) && ackcompare.IsNotNil(b.ko.Spec.TrainingJobDefinitions[index].EnableInterContainerTrafficEncryption) {
+					a.ko.Spec.TrainingJobDefinitions[index].EnableInterContainerTrafficEncryption = b.ko.Spec.TrainingJobDefinitions[index].EnableInterContainerTrafficEncryption
+				}
+				if ackcompare.IsNil(a.ko.Spec.TrainingJobDefinitions[index].EnableManagedSpotTraining) && ackcompare.IsNotNil(b.ko.Spec.TrainingJobDefinitions[index].EnableManagedSpotTraining) {
+					a.ko.Spec.TrainingJobDefinitions[index].EnableManagedSpotTraining = b.ko.Spec.TrainingJobDefinitions[index].EnableManagedSpotTraining
+				}
+				if ackcompare.IsNil(a.ko.Spec.TrainingJobDefinitions[index].EnableNetworkIsolation) && ackcompare.IsNotNil(b.ko.Spec.TrainingJobDefinitions[index].EnableNetworkIsolation) {
+					a.ko.Spec.TrainingJobDefinitions[index].EnableNetworkIsolation = b.ko.Spec.TrainingJobDefinitions[index].EnableNetworkIsolation
+				}
+			}
+		}
+	}
+
 }

pkg/resource/hyper_parameter_tuning_job/testdata/v1alpha1/readone/observed/completed_variation.yaml

@@ -109,7 +109,6 @@ spec:
       volumeSizeInGB: 25
     roleARN: arn:aws:iam::123456789012:role/service-role/AmazonSageMaker-ExecutionRole-20210920T111639
     staticHyperParameters:
-      _tuning_objective_metric: validation:error
       base_score: "0.5"
     stoppingCondition:
       maxRuntimeInSeconds: 3600