test: Minor improvements to test framework (#134)

Author: ellistarn

Makefile

@@ -90,4 +90,12 @@ build-deploy: ## Create a deployment file that can be applied with `kubectl appl
 ## Run e2e tests against cluster pointed to by ~/.kube/config
 .PHONY: e2etest
 e2etest:
-	cd test && go test -v ./... -count=1 --ginkgo.v
+	cd test && go test \
+		-p 1 \
+		-count 1 \
+		-timeout 60m \
+		-v \
+		./suites/... \
+		--ginkgo.focus="${FOCUS}" \
+		--ginkgo.timeout=60m \
+		--ginkgo.v

test/go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/aws/aws-sdk-go v1.42.18
 	github.com/imdario/mergo v0.3.13
 	github.com/onsi/ginkgo/v2 v2.9.1
-	github.com/onsi/gomega v1.27.3
+	github.com/onsi/gomega v1.27.4
 	github.com/samber/lo v1.37.0
 	go.uber.org/zap v1.24.0
 	k8s.io/api v0.26.2

test/go.sum

@@ -564,8 +564,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ
 github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
-github.com/onsi/gomega v1.27.3 h1:5VwIwnBY3vbBDOJrNtA4rVdiTZCsq9B5F12pvy1Drmk=
-github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw=
+github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
+github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=

test/pkg/test/framework.go

@@ -3,19 +3,17 @@ package test
 import (
 	"context"
 	"reflect"
-	"sync"
 	"time"
 
 	"github.com/aws/aws-application-networking-k8s/pkg/aws/services"
 	"github.com/aws/aws-application-networking-k8s/pkg/latticestore"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/vpclattice"
 	. "github.com/onsi/ginkgo/v2"
-	"github.com/onsi/gomega"
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/format"
-	"github.com/onsi/gomega/types"
 	"github.com/samber/lo"
+	"github.com/samber/lo/parallel"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -31,14 +29,13 @@ func init() {
 	format.MaxLength = 0
 }
 
+type TestObject struct {
+	Type     client.Object
+	ListType client.ObjectList
+}
+
 var (
-	CleanupTimeout  = 300 * time.Second
-	CreationTimeout = 120 * time.Second
-	TestObjects     = []struct {
-		Type     client.Object
-		ListType client.ObjectList
-	}{
-		// Must currently be deleted in order to avoid https://github.com/aws/aws-application-networking-k8s/issues/115
+	TestObjects = []TestObject{
 		{&v1.Service{}, &v1.ServiceList{}},
 		{&appsv1.Deployment{}, &appsv1.DeploymentList{}},
 		{&v1beta1.HTTPRoute{}, &v1beta1.HTTPRouteList{}},
@@ -59,7 +56,8 @@ func NewFramework(ctx context.Context) *Framework {
 		Client:        lo.Must(client.New(controllerruntime.GetConfigOrDie(), client.Options{Scheme: scheme})),
 		LatticeClient: services.NewDefaultLattice(session.Must(session.NewSession()), ""), // region is currently hardcoded
 	}
-	gomega.Default.SetDefaultEventuallyPollingInterval(time.Second * 1)
+	SetDefaultEventuallyTimeout(180 * time.Second)
+	SetDefaultEventuallyPollingInterval(1 * time.Second)
 	BeforeEach(func() { framework.ExpectToBeClean(ctx) })
 	AfterSuite(func() { framework.ExpectToClean(ctx) })
 	return framework
@@ -68,35 +66,31 @@ func NewFramework(ctx context.Context) *Framework {
 func (env *Framework) ExpectToBeClean(ctx context.Context) {
 	Logger(ctx).Info("Expecting the test environment to be clean")
 	// Kubernetes API Objects
-	for _, testObject := range TestObjects {
-		env.EventuallyExpectNoneFound(ctx, testObject.ListType).WithOffset(1).Should(Succeed())
-	}
+	parallel.ForEach(TestObjects, func(testObject TestObject, _ int) {
+		defer GinkgoRecover()
+		env.EventuallyExpectNoneFound(ctx, testObject.ListType)
+	})
+
 	// AWS API Objects
 	Eventually(func(g Gomega) {
 		g.Expect(env.LatticeClient.ListServicesWithContext(ctx, &vpclattice.ListServicesInput{})).To(HaveField("Items", BeEmpty()))
 		g.Expect(env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{})).To(HaveField("Items", BeEmpty()))
 		g.Expect(env.LatticeClient.ListTargetGroupsWithContext(ctx, &vpclattice.ListTargetGroupsInput{})).To(HaveField("Items", BeEmpty()))
-	})
+	}).Should(Succeed())
 }
 
 func (env *Framework) ExpectToClean(ctx context.Context) {
 	Logger(ctx).Info("Cleaning the test environment")
-	wg := sync.WaitGroup{}
-	namespaces := &v1.NamespaceList{}
 	// Kubernetes API Objects
+	namespaces := &v1.NamespaceList{}
 	Expect(env.List(ctx, namespaces)).WithOffset(1).To(Succeed())
 	for _, namespace := range namespaces.Items {
-		for _, object := range TestObjects {
-			wg.Add(1)
-			go func(object client.Object, objectList client.ObjectList, namespace string) {
-				defer wg.Done()
-				defer GinkgoRecover()
-				env.ExpectDeleteAllToSucceed(ctx, object, namespace)
-				env.EventuallyExpectNoneFound(ctx, objectList).Should(Succeed())
-			}(object.Type.DeepCopyObject().(client.Object), object.ListType.DeepCopyObject().(client.ObjectList), namespace.Name)
-		}
+		parallel.ForEach(TestObjects, func(testObject TestObject, _ int) {
+			defer GinkgoRecover()
+			env.ExpectDeleteAllToSucceed(ctx, testObject.Type, namespace.Name)
+			env.EventuallyExpectNoneFound(ctx, testObject.ListType)
+		})
 	}
-	wg.Wait()
 
 	// AWS API Objects
 	// Delete Services
@@ -167,19 +161,19 @@ func (env *Framework) ExpectDeleteAllToSucceed(ctx context.Context, object clien
 	Expect(env.DeleteAllOf(ctx, object, client.InNamespace(namespace), client.HasLabels([]string{DiscoveryLabel}))).WithOffset(1).To(Succeed())
 }
 
-func (env *Framework) EventuallyExpectNotFound(ctx context.Context, objects ...client.Object) types.AsyncAssertion {
-	return Eventually(func(g Gomega) {
+func (env *Framework) EventuallyExpectNotFound(ctx context.Context, objects ...client.Object) {
+	Eventually(func(g Gomega) {
 		for _, object := range objects {
 			g.Expect(errors.IsNotFound(env.Get(ctx, client.ObjectKeyFromObject(object), object))).To(BeTrue())
 		}
-	}, CleanupTimeout)
+	}).Should(Succeed())
 }
 
-func (env *Framework) EventuallyExpectNoneFound(ctx context.Context, objectList client.ObjectList) types.AsyncAssertion {
-	return Eventually(func(g Gomega) {
+func (env *Framework) EventuallyExpectNoneFound(ctx context.Context, objectList client.ObjectList) {
+	Eventually(func(g Gomega) {
 		g.Expect(env.List(ctx, objectList, client.HasLabels([]string{DiscoveryLabel}))).To(Succeed())
 		g.Expect(meta.ExtractList(objectList)).To(HaveLen(0), "Expected to not find any %q with label %q", reflect.TypeOf(objectList), DiscoveryLabel)
-	}, CleanupTimeout)
+	}).WithOffset(1).Should(Succeed())
 }
 
 func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *v1beta1.Gateway) *vpclattice.ServiceNetworkSummary {
@@ -194,7 +188,7 @@ func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *v1beta1.Ga
 			}
 		}
 		g.Expect(found).ToNot(BeNil())
-	}, CreationTimeout).WithOffset(1).Should(Succeed())
+	}).WithOffset(1).Should(Succeed())
 	return found
 }
 
@@ -211,7 +205,7 @@ func (env *Framework) GetService(ctx context.Context, httpRoute *v1beta1.HTTPRou
 		}
 		g.Expect(found).ToNot(BeNil())
 		g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.ServiceStatusActive)))
-	}, CreationTimeout).WithOffset(1).Should(Succeed())
+	}).WithOffset(1).Should(Succeed())
 
 	return found
 }
@@ -229,7 +223,7 @@ func (env *Framework) GetTargetGroup(ctx context.Context, service *v1.Service) *
 		}
 		g.Expect(found).ToNot(BeNil())
 		g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.TargetGroupStatusActive)))
-	}, CreationTimeout).WithOffset(1).Should(Succeed())
+	}).WithOffset(1).Should(Succeed())
 	return found
 }
 
@@ -245,10 +239,12 @@ func (env *Framework) GetTargets(ctx context.Context, targetGroup *vpclattice.Ta
 		g.Expect(listTargetsOutput.Items).To(HaveLen(int(*deployment.Spec.Replicas)))
 
 		podIps := lo.Map(podList.Items, func(pod v1.Pod, _ int) string { return pod.Status.PodIP })
-		targetIps := lo.Filter(listTargetsOutput.Items, func(target *vpclattice.TargetSummary, _ int) bool { return lo.Contains(podIps, *target.Id) })
+		targetIps := lo.Filter(listTargetsOutput.Items, func(target *vpclattice.TargetSummary, _ int) bool {
+			return *target.Status == vpclattice.TargetStatusInitial && lo.Contains(podIps, *target.Id)
+		})
 		g.Expect(targetIps).To(HaveLen(int(*deployment.Spec.Replicas)))
 
 		found = listTargetsOutput.Items
-	}, CreationTimeout).WithOffset(1).Should(Succeed())
+	}).WithOffset(1).Should(Succeed())
 	return found
 }

test/suites/integration/httproute_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/aws/aws-application-networking-k8s/pkg/latticestore"
 	"github.com/aws/aws-application-networking-k8s/test/pkg/test"
+	"github.com/aws/aws-sdk-go/service/vpclattice"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
@@ -66,7 +67,7 @@ var _ = Describe("HTTPRoute", func() {
 			})
 
 			// Create Kubernetes API Objects
-			env.ExpectCreated(ctx,
+			framework.ExpectCreated(ctx,
 				test.Gateway,
 				httpRoute,
 				serviceV1,
@@ -78,41 +79,43 @@ var _ = Describe("HTTPRoute", func() {
 			// Verify AWS API Objects
 
 			// (reverse order to allow dependency propagation)
-			targetGroupV1 := env.GetTargetGroup(ctx, serviceV1)
-			targetsV1 := env.GetTargets(ctx, targetGroupV1, deploymentV1)
+			targetGroupV1 := framework.GetTargetGroup(ctx, serviceV1)
+			targetsV1 := framework.GetTargets(ctx, targetGroupV1, deploymentV1)
 			Expect(*targetGroupV1.VpcIdentifier).To(Equal(os.Getenv("CLUSTER_VPC_ID")))
 			Expect(*targetGroupV1.Protocol).To(Equal("HTTP"))                                         // TODO(liwenwu) should this be TCP?
 			Expect(*targetGroupV1.Port).To(BeEquivalentTo(serviceV1.Spec.Ports[0].TargetPort.IntVal)) // TODO(liwenwu) should this be .Spec.Port[0].Port?
 			for _, target := range targetsV1 {
 				Expect(*target.Port).To(BeEquivalentTo(serviceV1.Spec.Ports[0].TargetPort.IntVal))
-				Expect(*target.Status).To(BeEquivalentTo("UNUSED")) // TODO(liwenwu) should this be ACTIVE?
+				Expect(*target.Status).To(Equal(vpclattice.TargetStatusInitial)) // TODO(liwenwu) should this be HEALTHY?
 			}
 
-			targetGroupV2 := env.GetTargetGroup(ctx, serviceV2)
-			targetsV2 := env.GetTargets(ctx, targetGroupV2, deploymentV2)
+			targetGroupV2 := framework.GetTargetGroup(ctx, serviceV2)
+			targetsV2 := framework.GetTargets(ctx, targetGroupV2, deploymentV2)
 			Expect(*targetGroupV2.VpcIdentifier).To(Equal(os.Getenv("CLUSTER_VPC_ID")))
 			Expect(*targetGroupV2.Protocol).To(Equal("HTTP"))                                         // TODO(liwenwu) should this be TCP?
 			Expect(*targetGroupV2.Port).To(BeEquivalentTo(serviceV2.Spec.Ports[0].TargetPort.IntVal)) // TODO(liwenwu) should this be .Spec.Port[0].Port?
 			for _, target := range targetsV2 {
 				Expect(*target.Port).To(BeEquivalentTo(serviceV2.Spec.Ports[0].TargetPort.IntVal))
-				Expect(*target.Status).To(BeEquivalentTo("UNUSED")) // TODO(liwenwu) should this be ACTIVE?
+				Expect(*target.Status).To(Equal(vpclattice.TargetStatusInitial)) // TODO(liwenwu) should this be HEALTHY?
 			}
 
-			service := env.GetService(ctx, httpRoute)
+			service := framework.GetService(ctx, httpRoute)
 			Expect(*service.DnsEntry).To(ContainSubstring(latticestore.AWSServiceName(httpRoute.Name, httpRoute.Namespace))) // TODO(liwenwu) is there something else we should verify about service?
 
-			serviceNetwork := env.GetServiceNetwork(ctx, test.Gateway)
+			serviceNetwork := framework.GetServiceNetwork(ctx, test.Gateway)
 			Expect(*serviceNetwork.NumberOfAssociatedServices).To(BeEquivalentTo(1))
 			Expect(*serviceNetwork.NumberOfAssociatedVPCs).To(BeEquivalentTo(0)) // TODO(liwenwu) should this be 1?
 
 			// Cleanup Kubernetes API Objects
-			Expect(env.Delete(ctx, test.Gateway)).To(Succeed())
-			Expect(env.Delete(ctx, httpRoute)).To(Succeed())
-			Expect(env.Delete(ctx, serviceV1)).To(Succeed())
-			Expect(env.Delete(ctx, deploymentV1)).To(Succeed())
-			Expect(env.Delete(ctx, serviceV2)).To(Succeed())
-			Expect(env.Delete(ctx, deploymentV2)).To(Succeed())
-			env.ExpectToBeClean(ctx)
+			framework.ExpectDeleted(ctx,
+				test.Gateway,
+				httpRoute,
+				serviceV1,
+				deploymentV1,
+				serviceV2,
+				deploymentV2,
+			)
+			framework.ExpectToBeClean(ctx)
 		})
 	})
 })

test/suites/integration/suite_test.go

@@ -9,12 +9,12 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var env *test.Framework
+var framework *test.Framework
 var ctx context.Context
 
 func TestIntegration(t *testing.T) {
 	ctx = test.NewContext(t)
-	env = test.NewFramework(ctx)
+	framework = test.NewFramework(ctx)
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Integration")
 }