Add e2e traffic test Setup (#215)

* - Improved Test framework, adding VPC Lattice resource thorough clean up logic in the AfterSuite()

- Trace each test run created k8s and VPC Lattice resource, properly clean up them in AfterSuite()

- Add more retry and wait by Eventually(), base on vpc lattice api throttling setting and async api time spent

- Add HeaderMatch httproute integ-test case

* Delete header_match_httproute test case

* Add traffic test setup in the test framework

* Add LatticeAssignedDomainName const

---------

Co-authored-by: Zijun Wang <zijunw@amazon.com>

Author: zijun726911

controllers/httproute_controller.go

@@ -63,7 +63,8 @@ type HTTPRouteReconciler struct {
 }
 
 const (
-	httpRouteFinalizer = "httproute.k8s.aws/resources"
+	httpRouteFinalizer        = "httproute.k8s.aws/resources"
+	LatticeAssignedDomainName = "application-networking.k8s.aws/lattice-assigned-domain-name"
 )
 
 func NewHttpRouteReconciler(cloud aws.Cloud, client client.Client, scheme *runtime.Scheme, eventRecorder record.EventRecorder,
@@ -274,7 +275,7 @@ func (r *HTTPRouteReconciler) updateHTTPRouteStatus(ctx context.Context, dns str
 		httproute.ObjectMeta.Annotations = make(map[string]string)
 	}
 
-	httproute.ObjectMeta.Annotations["application-networking.k8s.aws/lattice-assigned-domain-name"] = dns
+	httproute.ObjectMeta.Annotations[LatticeAssignedDomainName] = dns
 
 	if err := r.Client.Patch(ctx, httproute, client.MergeFrom(httprouteOld)); err != nil {
 		glog.V(2).Infof("updateHTTPRouteStatus: Patch() received err %v \n", err)

test/go.mod

@@ -52,6 +52,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

test/go.sum

@@ -90,6 +90,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
@@ -164,6 +165,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -509,6 +511,7 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=

test/pkg/test/framework.go

@@ -2,10 +2,14 @@ package test
 
 import (
 	"context"
+	"github.com/aws/aws-application-networking-k8s/controllers"
 	"github.com/aws/aws-application-networking-k8s/pkg/config"
 	"github.com/aws/aws-application-networking-k8s/pkg/model/lattice"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
 	"log"
 	"os"
 	"reflect"
@@ -51,6 +55,9 @@ var (
 
 type Framework struct {
 	client.Client
+	ctx                                 context.Context
+	k8sScheme                           *runtime.Scheme
+	controllerRuntimeConfig             *rest.Config
 	LatticeClient                       services.Lattice
 	TestCasesCreatedServiceNetworkNames map[string]bool //key: ServiceNetworkName; value: not in use, meaningless
 	TestCasesCreatedServiceNames        map[string]bool //key: ServiceName; value not in use, meaningless
@@ -61,9 +68,13 @@ func NewFramework(ctx context.Context) *Framework {
 	var scheme = scheme.Scheme
 	lo.Must0(v1beta1.Install(scheme))
 	lo.Must0(v1alpha1.Install(scheme))
+	controllerRuntimeConfig := controllerruntime.GetConfigOrDie()
 	framework := &Framework{
-		Client:                              lo.Must(client.New(controllerruntime.GetConfigOrDie(), client.Options{Scheme: scheme})),
+		Client:                              lo.Must(client.New(controllerRuntimeConfig, client.Options{Scheme: scheme})),
 		LatticeClient:                       services.NewDefaultLattice(session.Must(session.NewSession()), config.Region), // region is currently hardcoded
+		ctx:                                 ctx,
+		k8sScheme:                           scheme,
+		controllerRuntimeConfig:             controllerRuntimeConfig,
 		TestCasesCreatedServiceNetworkNames: make(map[string]bool),
 		TestCasesCreatedServiceNames:        make(map[string]bool),
 		TestCasesCreatedTargetGroupNames:    make(map[string]bool),
@@ -471,3 +482,10 @@ func (env *Framework) DeleteAllFrameworkTracedTargetGroups(ctx aws.Context) {
 	}
 	env.TestCasesCreatedServiceNames = make(map[string]bool)
 }
+
+func (env *Framework) GetVpcLatticeServiceDns(httpRouteName string, httpRouteNamespace string) string {
+	httproute := v1beta1.HTTPRoute{}
+	env.Get(env.ctx, types.NamespacedName{Name: httpRouteName, Namespace: httpRouteNamespace}, &httproute)
+	vpcLatticeServiceDns := httproute.Annotations[controllers.LatticeAssignedDomainName]
+	return vpcLatticeServiceDns
+}

test/pkg/test/httpapp.go

@@ -44,7 +44,7 @@ func (env *Framework) NewHttpApp(options HTTPAppOptions) (*appsv1.Deployment, *v
 						Image: "public.ecr.aws/x2j8p8w7/http-server:latest",
 						Env: []v1.EnvVar{{
 							Name:  "PodName",
-							Value: options.Name,
+							Value: options.Name + " handler pod",
 						}},
 					}},
 				},

test/pkg/test/pod_manager.go

@@ -0,0 +1,90 @@
+package test
+
+import (
+	"bytes"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/remotecommand"
+	"log"
+	"net/http"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+)
+
+// https://github.com/aws/amazon-vpc-cni-k8s/blob/7eeb2a9ab437887f77de30a5eab20bb42742df06/test/framework/resources/k8s/resources/pod.go#L188
+func (env *Framework) PodExec(namespace string, podName string, cmd string) (string, string, error) {
+	log.Printf("PodExec() [namespace: %v] [podName: %v] [command: %v] \n", namespace, podName, cmd)
+	restClient, err := env.getRestClientForPod(namespace, podName)
+	if err != nil {
+		return "", "", err
+	}
+	command := []string{
+		"sh",
+		"-c",
+		cmd,
+	}
+	execOptions := &v1.PodExecOptions{
+		Stdout:  true,
+		Stderr:  true,
+		Command: command,
+	}
+
+	restClient.Get()
+	req := restClient.Post().
+		Resource("pods").
+		Name(podName).
+		Namespace(namespace).
+		SubResource("exec").
+		VersionedParams(execOptions, runtime.NewParameterCodec(scheme.Scheme))
+
+	exec, err := remotecommand.NewSPDYExecutor(env.controllerRuntimeConfig, http.MethodPost, req.URL())
+	if err != nil {
+		return "", "", err
+	}
+
+	var stdout, stderr bytes.Buffer
+	err = exec.StreamWithContext(env.ctx, remotecommand.StreamOptions{
+		Stdout: &stdout,
+		Stderr: &stderr,
+	})
+	stdoutStr := stdout.String()
+	stderrStr := stderr.String()
+	log.Println("stdout: ", stdoutStr)
+	log.Println("stderr: ", stderrStr)
+	log.Println("err: ", err)
+	return stdoutStr, stderrStr, err
+}
+
+func (env *Framework) GetPodsByDeploymentName(deploymentName string, deploymentNamespce string) []v1.Pod {
+	deployment := appsv1.Deployment{}
+	env.Get(env.ctx, types.NamespacedName{Name: deploymentName, Namespace: deploymentNamespce}, &deployment)
+	pods := &v1.PodList{}
+	log.Println("deployment.Spec.Selector.MatchLabels:", deployment.Spec.Selector.MatchLabels)
+	env.List(env.ctx, pods, client.MatchingLabelsSelector{
+		Selector: labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels),
+	})
+	return pods.Items
+}
+
+func (env *Framework) getRestClientForPod(namespace string, name string) (rest.Interface, error) {
+	pod := &v1.Pod{}
+	err := env.Get(env.ctx, types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}, pod)
+	if err != nil {
+		return nil, err
+	}
+
+	gkv, err := apiutil.GVKForObject(pod, env.k8sScheme)
+	if err != nil {
+		return nil, err
+	}
+	return apiutil.RESTClientForGVK(gkv, false, env.controllerRuntimeConfig, serializer.NewCodecFactory(env.k8sScheme))
+}

test/suites/integration/httproute_test.go

@@ -1,13 +1,15 @@
 package integration
 
 import (
+	"fmt"
 	"github.com/aws/aws-application-networking-k8s/pkg/latticestore"
 	"github.com/aws/aws-application-networking-k8s/test/pkg/test"
 	"github.com/aws/aws-sdk-go/service/vpclattice"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"log"
 	"os"
 	"time"
@@ -100,18 +102,38 @@ var _ = Describe("HTTPRoute", func() {
 				httprouteRules := pathMatchHttpRoute.Spec.Rules
 
 				g.Expect(err).To(BeNil())
-				log.Println("*httprouteRules[0].Matches[0].Path.Value", *(httprouteRules[0].Matches[0].Path.Value))
-				log.Println("*rule0.Match.HttpMatch.PathMatch.Match.Prefix", *(rule0.Match.HttpMatch.PathMatch.Match.Prefix))
-
-				g.Expect([]string{
+				retrievedRules := []string{
 					*rule0.Match.HttpMatch.PathMatch.Match.Prefix,
-					*rule1.Match.HttpMatch.PathMatch.Match.Prefix}).To(
-					ContainElements(
-						*httprouteRules[0].Matches[0].Path.Value,
-						*httprouteRules[1].Matches[0].Path.Value))
+					*rule1.Match.HttpMatch.PathMatch.Match.Prefix}
+				expectedRules := []string{*httprouteRules[0].Matches[0].Path.Value,
+					*httprouteRules[1].Matches[0].Path.Value}
+				log.Println("retrievedRules", retrievedRules)
+				log.Println("expectedRules", expectedRules)
+
+				g.Expect(retrievedRules).To(
+					ContainElements(expectedRules))
 			}).WithOffset(1).Should(Succeed())
+			time.Sleep(30 * time.Second) //Need to wait for config propagate to VPC lattice dataplane
+
+			log.Println("Verifying traffic")
+			dnsName := testFramework.GetVpcLatticeServiceDns(pathMatchHttpRoute.Name, pathMatchHttpRoute.Namespace)
+
+			testFramework.Get(ctx, types.NamespacedName{Name: deployment1.Name, Namespace: deployment1.Namespace}, deployment1)
 
-			//TODO: test traffic in integ-test https://stackoverflow.com/questions/43314689/example-of-exec-in-k8ss-pod-by-using-go-client
+			//get the pods of deployment1
+			pods := testFramework.GetPodsByDeploymentName(deployment1.Name, deployment1.Namespace)
+			Expect(len(pods)).To(BeEquivalentTo(1))
+			log.Println("pods[0].Name:", pods[0].Name)
+
+			cmd1 := fmt.Sprintf("curl %s/pathmatch0", dnsName)
+			stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd1)
+			Expect(err).To(BeNil())
+			Expect(stdout).To(ContainSubstring("test-v1 handler pod"))
+
+			cmd2 := fmt.Sprintf("curl %s/pathmatch1", dnsName)
+			stdout, _, err = testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd2)
+			Expect(err).To(BeNil())
+			Expect(stdout).To(ContainSubstring("test-v2 handler pod"))
 
 			testFramework.ExpectDeleted(ctx,
 				gateway,
@@ -130,6 +152,5 @@ var _ = Describe("HTTPRoute", func() {
 				deployment2)
 
 		})
-
 	})
 })