Improve Test framework, Optimize resource clean up and 'ExpectToBeClean' logic (#214)

* - Improved Test framework, adding VPC Lattice resource thorough clean up logic in the AfterSuite()

- Trace each test run created k8s and VPC Lattice resource, properly clean up them in AfterSuite()

- Add more retry and wait by Eventually(), base on vpc lattice api throttling setting and async api time spent

- Add HeaderMatch httproute integ-test case

* Delete header_match_httproute test case

* - Add method: `AfterEach(func() {framework.ExpectToBeClean(ctx)})`.

 in `ExpectToBeClean`, it should expect only current cluster's resource is clean

---------

Co-authored-by: Zijun Wang <zijunw@amazon.com>

Author: zijun726911

pkg/config/controller_config.go

@@ -16,7 +16,7 @@ const (
 // TODO endpoint, region
 var VpcID = "vpc-xxxx"
 var AccountID = "yyyyyy"
-var Region = "us-east-2"
+var Region = "us-west-2"
 var logLevel = defaultLogLevel
 
 func GetLogLevel() string {

pkg/gateway/model_build_lattice_service.go

@@ -139,7 +139,7 @@ func (t *latticeServiceModelBuildTask) buildLatticeService(ctx context.Context)
 		glog.V(2).Infof("Setting customer-domain-name: %v for httpRoute %v-%v",
 			spec.CustomerDomainName, t.httpRoute.Name, t.httpRoute.Namespace)
 	} else {
-		glog.V(2).Infof("No customter-domain-name for httproute :%v-%v",
+		glog.V(2).Infof("No custom-domain-name for httproute :%v-%v",
 			t.httpRoute.Name, t.httpRoute.Namespace)
 		spec.CustomerDomainName = ""
 	}

test/pkg/test/framework.go

@@ -1,16 +1,29 @@
 package test
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/gateway-api/apis/v1beta1"
 )
 
-var Gateway = New(&v1beta1.Gateway{
-	Spec: v1beta1.GatewaySpec{
-		GatewayClassName: "amazon-vpc-lattice",
-		Listeners: []v1beta1.Listener{{
-			Name:     "http",
-			Protocol: v1beta1.HTTPProtocolType,
-			Port:     80,
-		}},
-	},
-})
+func (env *Framework) NewGateway() *v1beta1.Gateway {
+	gateway := New(
+		&v1beta1.Gateway{
+			ObjectMeta: metav1.ObjectMeta{
+				Annotations: map[string]string{
+					"application-networking.k8s.aws/lattice-vpc-association": "true",
+				},
+			},
+			Spec: v1beta1.GatewaySpec{
+				GatewayClassName: "amazon-vpc-lattice",
+				Listeners: []v1beta1.Listener{{
+					Name:     "http",
+					Protocol: v1beta1.HTTPProtocolType,
+					Port:     80,
+				}},
+			},
+			Status: v1beta1.GatewayStatus{},
+		},
+	)
+	env.TestCasesCreatedServiceNetworkNames[gateway.Name] = true
+	return gateway
+}

test/pkg/test/gateway.go

@@ -1,6 +1,7 @@
 package test
 
 import (
+	"github.com/aws/aws-application-networking-k8s/pkg/latticestore"
 	"github.com/samber/lo"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -16,50 +17,55 @@ type HTTPAppOptions struct {
 	MergeFromService    []*v1.Service
 }
 
-func HTTPApp(options HTTPAppOptions) (*appsv1.Deployment, *v1.Service) {
+func (env *Framework) NewHttpApp(options HTTPAppOptions) (*appsv1.Deployment, *v1.Service) {
 	if options.Port == 0 {
 		options.Port = 80
 	}
 	if options.TargetPort == 0 {
 		options.TargetPort = 8090
 	}
-	return New(&appsv1.Deployment{
-			Spec: appsv1.DeploymentSpec{
-				Replicas: lo.ToPtr(int32(1)),
-				Selector: &metav1.LabelSelector{
-					MatchLabels: map[string]string{
+	deployment := New(&appsv1.Deployment{
+		Spec: appsv1.DeploymentSpec{
+			Replicas: lo.ToPtr(int32(1)),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": options.Name,
+				},
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
 						"app": options.Name,
 					},
 				},
-				Template: v1.PodTemplateSpec{
-					ObjectMeta: metav1.ObjectMeta{
-						Labels: map[string]string{
-							"app": options.Name,
-						},
-					},
-					Spec: v1.PodSpec{
-						Containers: []v1.Container{{
-							Name:  options.Name,
-							Image: "public.ecr.aws/x2j8p8w7/http-server:latest",
-							Env: []v1.EnvVar{{
-								Name:  "PodName",
-								Value: options.Name,
-							}},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{{
+						Name:  options.Name,
+						Image: "public.ecr.aws/x2j8p8w7/http-server:latest",
+						Env: []v1.EnvVar{{
+							Name:  "PodName",
+							Value: options.Name,
 						}},
-					},
+					}},
 				},
 			},
-		}, options.MergeFromDeployment...),
-		New(&v1.Service{
-			Spec: v1.ServiceSpec{
-				Selector: map[string]string{
-					"app": options.Name,
-				},
-				Ports: []v1.ServicePort{{
-					Protocol:   v1.ProtocolTCP,
-					Port:       int32(options.Port),
-					TargetPort: intstr.FromInt(options.TargetPort),
-				}},
+		},
+	}, options.MergeFromDeployment...)
+
+	service := New(&v1.Service{
+		Spec: v1.ServiceSpec{
+			Selector: map[string]string{
+				"app": options.Name,
 			},
-		}, options.MergeFromService...)
+			Ports: []v1.ServicePort{{
+				Protocol:   v1.ProtocolTCP,
+				Port:       int32(options.Port),
+				TargetPort: intstr.FromInt(options.TargetPort),
+			}},
+		},
+	}, options.MergeFromService...)
+	env.TestCasesCreatedTargetGroupNames[latticestore.TargetGroupName(service.Name, service.Namespace)] = true
+
+	return deployment, service
+
 }

test/pkg/test/httpapp.go

@@ -16,7 +16,7 @@ var (
 	sequentialNumber     = 0
 	randomizer           = rand.New(rand.NewSource(time.Now().UnixNano())) //nolint
 	sequentialNumberLock = new(sync.Mutex)
-	DiscoveryLabel       = "testing.kubernetes.io"
+	DiscoveryLabel       = "testing.kubernetes.io-" + randomdata.Alphanumeric(8) // Each time test suite run have a different label for k8s resource
 )
 
 func New[T client.Object](t T, mergeFrom ...T) T {

test/pkg/test/metadata.go

@@ -0,0 +1,48 @@
+package test
+
+import (
+	"github.com/aws/aws-application-networking-k8s/pkg/latticestore"
+	"github.com/samber/lo"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/gateway-api/apis/v1beta1"
+	"strconv"
+)
+
+func (env *Framework) NewPathMatchHttpRoute(parentRefsGateway *v1beta1.Gateway, services []*v1.Service) *v1beta1.HTTPRoute {
+	var rules []v1beta1.HTTPRouteRule
+	for i, service := range services {
+		rule := v1beta1.HTTPRouteRule{
+			BackendRefs: []v1beta1.HTTPBackendRef{{
+				BackendRef: v1beta1.BackendRef{
+					BackendObjectReference: v1beta1.BackendObjectReference{
+						Name: v1beta1.ObjectName(service.Name),
+						Kind: lo.ToPtr(v1beta1.Kind("Service")),
+					},
+				},
+			}},
+			Matches: []v1beta1.HTTPRouteMatch{
+				{
+					Path: &v1beta1.HTTPPathMatch{
+						Type:  lo.ToPtr(v1beta1.PathMatchPathPrefix),
+						Value: lo.ToPtr("/pathmatch" + strconv.Itoa(i)),
+					},
+				},
+			},
+		}
+		rules = append(rules, rule)
+	}
+	httpRoute := New(&v1beta1.HTTPRoute{
+		Spec: v1beta1.HTTPRouteSpec{
+			CommonRouteSpec: v1beta1.CommonRouteSpec{
+				ParentRefs: []v1beta1.ParentReference{{
+					Name:        v1beta1.ObjectName(parentRefsGateway.Name),
+					SectionName: lo.ToPtr(v1beta1.SectionName("http")),
+				}},
+			},
+			Rules: rules,
+		},
+	})
+
+	env.TestCasesCreatedServiceNames[latticestore.AWSServiceName(httpRoute.Name, httpRoute.Namespace)] = true
+	return httpRoute
+}