|
| 1 | +package test |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "reflect" |
| 6 | + "sync" |
| 7 | + "time" |
| 8 | + |
| 9 | + "github.com/aws/aws-application-networking-k8s/pkg/aws/services" |
| 10 | + "github.com/aws/aws-application-networking-k8s/pkg/latticestore" |
| 11 | + "github.com/aws/aws-sdk-go/aws/session" |
| 12 | + "github.com/aws/aws-sdk-go/service/vpclattice" |
| 13 | + . "github.com/onsi/ginkgo/v2" |
| 14 | + "github.com/onsi/gomega" |
| 15 | + . "github.com/onsi/gomega" |
| 16 | + "github.com/onsi/gomega/format" |
| 17 | + "github.com/onsi/gomega/types" |
| 18 | + "github.com/samber/lo" |
| 19 | + appsv1 "k8s.io/api/apps/v1" |
| 20 | + v1 "k8s.io/api/core/v1" |
| 21 | + "k8s.io/apimachinery/pkg/api/errors" |
| 22 | + "k8s.io/apimachinery/pkg/api/meta" |
| 23 | + "k8s.io/client-go/kubernetes/scheme" |
| 24 | + controllerruntime "sigs.k8s.io/controller-runtime" |
| 25 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 26 | + "sigs.k8s.io/gateway-api/apis/v1beta1" |
| 27 | + "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" |
| 28 | +) |
| 29 | + |
| 30 | +func init() { |
| 31 | + format.MaxLength = 0 |
| 32 | +} |
| 33 | + |
| 34 | +var ( |
| 35 | + CleanupTimeout = 300 * time.Second |
| 36 | + CreationTimeout = 120 * time.Second |
| 37 | + TestObjects = []struct { |
| 38 | + Type client.Object |
| 39 | + ListType client.ObjectList |
| 40 | + }{ |
| 41 | + // Must currently be deleted in order to avoid https://github.com/aws/aws-application-networking-k8s/issues/115 |
| 42 | + {&v1.Service{}, &v1.ServiceList{}}, |
| 43 | + {&appsv1.Deployment{}, &appsv1.DeploymentList{}}, |
| 44 | + {&v1beta1.HTTPRoute{}, &v1beta1.HTTPRouteList{}}, |
| 45 | + {&v1beta1.Gateway{}, &v1beta1.GatewayList{}}, |
| 46 | + } |
| 47 | +) |
| 48 | + |
| 49 | +type Framework struct { |
| 50 | + client.Client |
| 51 | + LatticeClient services.Lattice |
| 52 | +} |
| 53 | + |
| 54 | +func NewFramework(ctx context.Context) *Framework { |
| 55 | + var scheme = scheme.Scheme |
| 56 | + lo.Must0(v1beta1.Install(scheme)) |
| 57 | + lo.Must0(v1alpha1.Install(scheme)) |
| 58 | + framework := &Framework{ |
| 59 | + Client: lo.Must(client.New(controllerruntime.GetConfigOrDie(), client.Options{Scheme: scheme})), |
| 60 | + LatticeClient: services.NewDefaultLattice(session.Must(session.NewSession()), ""), // region is currently hardcoded |
| 61 | + } |
| 62 | + gomega.Default.SetDefaultEventuallyPollingInterval(time.Second * 1) |
| 63 | + BeforeEach(func() { framework.ExpectToBeClean(ctx) }) |
| 64 | + AfterSuite(func() { framework.ExpectToClean(ctx) }) |
| 65 | + return framework |
| 66 | +} |
| 67 | + |
| 68 | +func (env *Framework) ExpectToBeClean(ctx context.Context) { |
| 69 | + Logger(ctx).Info("Expecting the test environment to be clean") |
| 70 | + // Kubernetes API Objects |
| 71 | + for _, testObject := range TestObjects { |
| 72 | + env.EventuallyExpectNoneFound(ctx, testObject.ListType).WithOffset(1).Should(Succeed()) |
| 73 | + } |
| 74 | + // AWS API Objects |
| 75 | + Eventually(func(g Gomega) { |
| 76 | + g.Expect(env.LatticeClient.ListServicesWithContext(ctx, &vpclattice.ListServicesInput{})).To(HaveField("Items", BeEmpty())) |
| 77 | + g.Expect(env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{})).To(HaveField("Items", BeEmpty())) |
| 78 | + g.Expect(env.LatticeClient.ListTargetGroupsWithContext(ctx, &vpclattice.ListTargetGroupsInput{})).To(HaveField("Items", BeEmpty())) |
| 79 | + }) |
| 80 | +} |
| 81 | + |
| 82 | +func (env *Framework) ExpectToClean(ctx context.Context) { |
| 83 | + Logger(ctx).Info("Cleaning the test environment") |
| 84 | + wg := sync.WaitGroup{} |
| 85 | + namespaces := &v1.NamespaceList{} |
| 86 | + // Kubernetes API Objects |
| 87 | + Expect(env.List(ctx, namespaces)).WithOffset(1).To(Succeed()) |
| 88 | + for _, namespace := range namespaces.Items { |
| 89 | + for _, object := range TestObjects { |
| 90 | + wg.Add(1) |
| 91 | + go func(object client.Object, objectList client.ObjectList, namespace string) { |
| 92 | + defer wg.Done() |
| 93 | + defer GinkgoRecover() |
| 94 | + env.ExpectDeleteAllToSucceed(ctx, object, namespace) |
| 95 | + env.EventuallyExpectNoneFound(ctx, objectList).Should(Succeed()) |
| 96 | + }(object.Type.DeepCopyObject().(client.Object), object.ListType.DeepCopyObject().(client.ObjectList), namespace.Name) |
| 97 | + } |
| 98 | + } |
| 99 | + wg.Wait() |
| 100 | + |
| 101 | + // AWS API Objects |
| 102 | + // Delete Services |
| 103 | + listServicesOutput, err := env.LatticeClient.ListServicesWithContext(ctx, &vpclattice.ListServicesInput{}) |
| 104 | + Expect(err).ToNot(HaveOccurred()) |
| 105 | + for _, service := range listServicesOutput.Items { |
| 106 | + // Delete ServiceNetworkServiceAssociations |
| 107 | + listServiceNetworkServiceAssociationsOutput, err := env.LatticeClient.ListServiceNetworkServiceAssociationsWithContext(ctx, &vpclattice.ListServiceNetworkServiceAssociationsInput{ServiceIdentifier: service.Id}) |
| 108 | + Expect(err).ToNot(HaveOccurred()) |
| 109 | + for _, serviceNetworkServiceAssociation := range listServiceNetworkServiceAssociationsOutput.Items { |
| 110 | + _, err := env.LatticeClient.DeleteServiceNetworkServiceAssociationWithContext(ctx, &vpclattice.DeleteServiceNetworkServiceAssociationInput{ServiceNetworkServiceAssociationIdentifier: serviceNetworkServiceAssociation.Id}) |
| 111 | + Expect(err).ToNot(HaveOccurred()) |
| 112 | + } |
| 113 | + // Delete Listeners |
| 114 | + listListenersOutput, err := env.LatticeClient.ListListenersWithContext(ctx, &vpclattice.ListListenersInput{ServiceIdentifier: service.Id}) |
| 115 | + Expect(err).ToNot(HaveOccurred()) |
| 116 | + for _, listener := range listListenersOutput.Items { |
| 117 | + _, err = env.LatticeClient.DeleteListenerWithContext(ctx, &vpclattice.DeleteListenerInput{ServiceIdentifier: service.Id, ListenerIdentifier: listener.Id}) |
| 118 | + Expect(err).ToNot(HaveOccurred()) |
| 119 | + } |
| 120 | + // Delete Service |
| 121 | + _, err = env.LatticeClient.DeleteServiceWithContext(ctx, &vpclattice.DeleteServiceInput{ServiceIdentifier: service.Id}) |
| 122 | + Expect(err).ToNot(HaveOccurred()) |
| 123 | + } |
| 124 | + // Delete TargetGroups |
| 125 | + listTargetGroupsOutput, err := env.LatticeClient.ListTargetGroupsWithContext(ctx, &vpclattice.ListTargetGroupsInput{}) |
| 126 | + Expect(err).ToNot(HaveOccurred()) |
| 127 | + for _, targetGroup := range listTargetGroupsOutput.Items { |
| 128 | + // Delete Targets |
| 129 | + listTargetsOutput, err := env.LatticeClient.ListTargetsWithContext(ctx, &vpclattice.ListTargetsInput{TargetGroupIdentifier: targetGroup.Id}) |
| 130 | + Expect(err).ToNot(HaveOccurred()) |
| 131 | + if targets := lo.Map(listTargetsOutput.Items, func(target *vpclattice.TargetSummary, _ int) *vpclattice.Target { |
| 132 | + return &vpclattice.Target{Id: target.Id} |
| 133 | + }); len(targets) > 0 { |
| 134 | + _, err = env.LatticeClient.DeregisterTargetsWithContext(ctx, &vpclattice.DeregisterTargetsInput{TargetGroupIdentifier: targetGroup.Id, Targets: targets}) |
| 135 | + Expect(err).ToNot(HaveOccurred()) |
| 136 | + } |
| 137 | + // Delete TargetGroup |
| 138 | + _, err = env.LatticeClient.DeleteTargetGroupWithContext(ctx, &vpclattice.DeleteTargetGroupInput{TargetGroupIdentifier: targetGroup.Id}) |
| 139 | + Expect(err).ToNot(HaveOccurred()) |
| 140 | + } |
| 141 | + listServiceNetworksOutput, err := env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{}) |
| 142 | + Expect(err).ToNot(HaveOccurred()) |
| 143 | + for _, serviceNetwork := range listServiceNetworksOutput.Items { |
| 144 | + _, err := env.LatticeClient.DeleteServiceNetworkWithContext(ctx, &vpclattice.DeleteServiceNetworkInput{ServiceNetworkIdentifier: serviceNetwork.Id}) |
| 145 | + Expect(err).ToNot(HaveOccurred()) |
| 146 | + } |
| 147 | + |
| 148 | + // Wait for objects to delete |
| 149 | + env.ExpectToBeClean(ctx) |
| 150 | +} |
| 151 | + |
| 152 | +func (env *Framework) ExpectCreated(ctx context.Context, objects ...client.Object) { |
| 153 | + for _, object := range objects { |
| 154 | + Logger(ctx).Infof("Creating %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) |
| 155 | + Expect(env.Create(ctx, object)).WithOffset(1).To(Succeed()) |
| 156 | + } |
| 157 | +} |
| 158 | + |
| 159 | +func (env *Framework) ExpectDeleted(ctx context.Context, objects ...client.Object) { |
| 160 | + for _, object := range objects { |
| 161 | + Logger(ctx).Infof("Deleting %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) |
| 162 | + Expect(env.Delete(ctx, object)).WithOffset(1).To(Succeed()) |
| 163 | + } |
| 164 | +} |
| 165 | + |
| 166 | +func (env *Framework) ExpectDeleteAllToSucceed(ctx context.Context, object client.Object, namespace string) { |
| 167 | + Expect(env.DeleteAllOf(ctx, object, client.InNamespace(namespace), client.HasLabels([]string{DiscoveryLabel}))).WithOffset(1).To(Succeed()) |
| 168 | +} |
| 169 | + |
| 170 | +func (env *Framework) EventuallyExpectNotFound(ctx context.Context, objects ...client.Object) types.AsyncAssertion { |
| 171 | + return Eventually(func(g Gomega) { |
| 172 | + for _, object := range objects { |
| 173 | + g.Expect(errors.IsNotFound(env.Get(ctx, client.ObjectKeyFromObject(object), object))).To(BeTrue()) |
| 174 | + } |
| 175 | + }, CleanupTimeout) |
| 176 | +} |
| 177 | + |
| 178 | +func (env *Framework) EventuallyExpectNoneFound(ctx context.Context, objectList client.ObjectList) types.AsyncAssertion { |
| 179 | + return Eventually(func(g Gomega) { |
| 180 | + g.Expect(env.List(ctx, objectList, client.HasLabels([]string{DiscoveryLabel}))).To(Succeed()) |
| 181 | + g.Expect(meta.ExtractList(objectList)).To(HaveLen(0), "Expected to not find any %q with label %q", reflect.TypeOf(objectList), DiscoveryLabel) |
| 182 | + }, CleanupTimeout) |
| 183 | +} |
| 184 | + |
| 185 | +func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *v1beta1.Gateway) *vpclattice.ServiceNetworkSummary { |
| 186 | + var found *vpclattice.ServiceNetworkSummary |
| 187 | + Eventually(func(g Gomega) { |
| 188 | + listServiceNetworksOutput, err := env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{}) |
| 189 | + g.Expect(err).ToNot(HaveOccurred()) |
| 190 | + g.Expect(listServiceNetworksOutput.Items).ToNot(BeEmpty()) |
| 191 | + for _, serviceNetwork := range listServiceNetworksOutput.Items { |
| 192 | + if lo.FromPtr(serviceNetwork.Name) == gateway.Name { |
| 193 | + found = serviceNetwork |
| 194 | + } |
| 195 | + } |
| 196 | + g.Expect(found).ToNot(BeNil()) |
| 197 | + }, CreationTimeout).WithOffset(1).Should(Succeed()) |
| 198 | + return found |
| 199 | +} |
| 200 | + |
| 201 | +func (env *Framework) GetService(ctx context.Context, httpRoute *v1beta1.HTTPRoute) *vpclattice.ServiceSummary { |
| 202 | + var found *vpclattice.ServiceSummary |
| 203 | + Eventually(func(g Gomega) { |
| 204 | + listServicesOutput, err := env.LatticeClient.ListServicesWithContext(ctx, &vpclattice.ListServicesInput{}) |
| 205 | + g.Expect(err).ToNot(HaveOccurred()) |
| 206 | + g.Expect(listServicesOutput.Items).ToNot(BeEmpty()) |
| 207 | + for _, service := range listServicesOutput.Items { |
| 208 | + if lo.FromPtr(service.Name) == latticestore.AWSServiceName(httpRoute.Name, httpRoute.Namespace) { |
| 209 | + found = service |
| 210 | + } |
| 211 | + } |
| 212 | + g.Expect(found).ToNot(BeNil()) |
| 213 | + g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.ServiceStatusActive))) |
| 214 | + }, CreationTimeout).WithOffset(1).Should(Succeed()) |
| 215 | + |
| 216 | + return found |
| 217 | +} |
| 218 | + |
| 219 | +func (env *Framework) GetTargetGroup(ctx context.Context, service *v1.Service) *vpclattice.TargetGroupSummary { |
| 220 | + var found *vpclattice.TargetGroupSummary |
| 221 | + Eventually(func(g Gomega) { |
| 222 | + listTargetGroupsOutput, err := env.LatticeClient.ListTargetGroupsWithContext(ctx, &vpclattice.ListTargetGroupsInput{}) |
| 223 | + g.Expect(err).ToNot(HaveOccurred()) |
| 224 | + g.Expect(listTargetGroupsOutput.Items).ToNot(BeEmpty()) |
| 225 | + for _, targetGroup := range listTargetGroupsOutput.Items { |
| 226 | + if lo.FromPtr(targetGroup.Name) == latticestore.TargetGroupName(service.Name, service.Namespace) { |
| 227 | + found = targetGroup |
| 228 | + } |
| 229 | + } |
| 230 | + g.Expect(found).ToNot(BeNil()) |
| 231 | + g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.TargetGroupStatusActive))) |
| 232 | + }, CreationTimeout).WithOffset(1).Should(Succeed()) |
| 233 | + return found |
| 234 | +} |
| 235 | + |
| 236 | +func (env *Framework) GetTargets(ctx context.Context, targetGroup *vpclattice.TargetGroupSummary, deployment *appsv1.Deployment) []*vpclattice.TargetSummary { |
| 237 | + var found []*vpclattice.TargetSummary |
| 238 | + Eventually(func(g Gomega) { |
| 239 | + podList := &v1.PodList{} |
| 240 | + g.Expect(env.List(ctx, podList, client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(Succeed()) |
| 241 | + g.Expect(podList.Items).To(HaveLen(int(*deployment.Spec.Replicas))) |
| 242 | + |
| 243 | + listTargetsOutput, err := env.LatticeClient.ListTargetsWithContext(ctx, &vpclattice.ListTargetsInput{TargetGroupIdentifier: targetGroup.Id}) |
| 244 | + g.Expect(err).ToNot(HaveOccurred()) |
| 245 | + g.Expect(listTargetsOutput.Items).To(HaveLen(int(*deployment.Spec.Replicas))) |
| 246 | + |
| 247 | + podIps := lo.Map(podList.Items, func(pod v1.Pod, _ int) string { return pod.Status.PodIP }) |
| 248 | + targetIps := lo.Filter(listTargetsOutput.Items, func(target *vpclattice.TargetSummary, _ int) bool { return lo.Contains(podIps, *target.Id) }) |
| 249 | + g.Expect(targetIps).To(HaveLen(int(*deployment.Spec.Replicas))) |
| 250 | + |
| 251 | + found = listTargetsOutput.Items |
| 252 | + }, CreationTimeout).WithOffset(1).Should(Succeed()) |
| 253 | + return found |
| 254 | +} |
0 commit comments