Skip to content

Commit ed90166

Browse files
mikhail-awsmikhail-aws
authored
add target group policy controller and status updates (#509)
* add target group policy controller and status updates
1 parent 61689bc commit ed90166

File tree

10 files changed

+259
-19
lines changed

10 files changed

+259
-19
lines changed

‎cmd/aws-application-networking-k8s/main.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,11 @@ func main() {
193193
setupLog.Fatalf("iam auth policy controller setup failed: %s", err)
194194
}
195195

196+
err = controllers.RegisterTargetGroupPolicyController(ctrlLog.Named("target-group-policy"), mgr)
197+
if err != nil {
198+
setupLog.Fatalf("target group policy controller setup failed: %s", err)
199+
}
200+
196201
err = controllers.RegisterVpcAssociationPolicyController(ctrlLog.Named("vpc-association-policy"), cloud, finalizerManager, mgr)
197202
if err != nil {
198203
setupLog.Fatalf("vpc association policy controller setup failed: %s", err)

‎config/crds/bases/application-networking.k8s.aws_targetgrouppolicies.yaml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,14 @@ spec:
157157
- targetRef
158158
type: object
159159
status:
160-
description: TargetGroupPolicyStatus defines the observed state of AccessLogPolicy.
160+
default:
161+
conditions:
162+
- lastTransitionTime: "1970-01-01T00:00:00Z"
163+
message: Waiting for controller
164+
reason: NotReconciled
165+
status: Unknown
166+
type: Accepted
167+
description: Status defines the current state of TargetGroupPolicy.
161168
properties:
162169
conditions:
163170
default:
@@ -254,4 +261,5 @@ spec:
254261
type: object
255262
served: true
256263
storage: true
257-
subresources: {}
264+
subresources:
265+
status: {}

‎controllers/eventhandlers/policy.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ func (h *policyEventHandler[T]) MapObjectToPolicy() handler.EventHandler {
2727

2828
func (h *policyEventHandler[T]) mapObjectToPolicy(ctx context.Context, eventObj client.Object) []reconcile.Request {
2929
var requests []reconcile.Request
30+
3031
policies, err := policyhelper.GetAttachedPolicies(ctx, h.client, k8s.NamespacedName(eventObj), *new(T))
3132
if err != nil {
3233
h.log.Errorf("Failed calling k8s operation: %s", err.Error())

‎controllers/targetgrouppolicy_controller.go‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
corev1 "k8s.io/api/core/v1"
8+
"k8s.io/apimachinery/pkg/api/meta"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/apimachinery/pkg/types"
11+
ctrl "sigs.k8s.io/controller-runtime"
12+
"sigs.k8s.io/controller-runtime/pkg/builder"
13+
"sigs.k8s.io/controller-runtime/pkg/client"
14+
"sigs.k8s.io/controller-runtime/pkg/handler"
15+
"sigs.k8s.io/controller-runtime/pkg/predicate"
16+
gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
17+
18+
anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1"
19+
"github.com/aws/aws-application-networking-k8s/pkg/k8s"
20+
"github.com/aws/aws-application-networking-k8s/pkg/k8s/policyhelper"
21+
"github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog"
22+
)
23+
24+
type TargetGroupPolicyController struct {
25+
log gwlog.Logger
26+
client client.Client
27+
}
28+
29+
func RegisterTargetGroupPolicyController(log gwlog.Logger, mgr ctrl.Manager) error {
30+
controller := &TargetGroupPolicyController{
31+
log: log,
32+
client: mgr.GetClient(),
33+
}
34+
mapfn := targetGroupPolicyMapFunc(mgr.GetClient(), log)
35+
return ctrl.NewControllerManagedBy(mgr).
36+
For(&anv1alpha1.TargetGroupPolicy{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
37+
Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(mapfn)).
38+
Complete(controller)
39+
}
40+
41+
func (c *TargetGroupPolicyController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
42+
tgPolicy := &anv1alpha1.TargetGroupPolicy{}
43+
err := c.client.Get(ctx, req.NamespacedName, tgPolicy)
44+
if err != nil {
45+
return ctrl.Result{}, client.IgnoreNotFound(err)
46+
}
47+
c.log.Infow("reconcile target group policy", "req", req, "targetRef", tgPolicy.Spec.TargetRef)
48+
49+
validationErr := c.validateSpec(ctx, tgPolicy)
50+
reason := validationErrToStatusReason(validationErr)
51+
msg := ""
52+
if validationErr != nil {
53+
msg = validationErr.Error()
54+
}
55+
c.updatePolicyCondition(tgPolicy, reason, msg)
56+
err = c.client.Status().Update(ctx, tgPolicy)
57+
if err != nil {
58+
return ctrl.Result{}, err
59+
}
60+
61+
c.log.Infow("reconciled target group policy",
62+
"req", req,
63+
"targetRef", tgPolicy.Spec.TargetRef,
64+
)
65+
return ctrl.Result{}, nil
66+
}
67+
68+
func (c *TargetGroupPolicyController) validateSpec(ctx context.Context, tgPolicy *anv1alpha1.TargetGroupPolicy) error {
69+
tr := tgPolicy.Spec.TargetRef
70+
if tr.Group != corev1.GroupName {
71+
return fmt.Errorf("%w: %s", GroupNameError, tr.Group)
72+
}
73+
if string(tr.Kind) != "Service" {
74+
return fmt.Errorf("%w: %s", KindError, tr.Kind)
75+
}
76+
tgref := types.NamespacedName{
77+
Namespace: tgPolicy.Namespace,
78+
Name: string(tgPolicy.Spec.TargetRef.Name),
79+
}
80+
valid, err := policyhelper.GetValidPolicy(ctx, c.client, tgref, tgPolicy)
81+
if err != nil {
82+
return nil
83+
}
84+
if valid != nil && valid.GetNamespacedName() != tgPolicy.GetNamespacedName() {
85+
return fmt.Errorf("%w, with policy %s", TargetRefConflict, valid.GetName())
86+
}
87+
refExists, err := c.targetRefExists(ctx, tgPolicy)
88+
if err != nil {
89+
return err
90+
}
91+
if !refExists {
92+
return fmt.Errorf("%w: %s", TargetRefNotFound, tr.Name)
93+
}
94+
return nil
95+
}
96+
97+
func (c *TargetGroupPolicyController) targetRefExists(ctx context.Context, tgPolicy *anv1alpha1.TargetGroupPolicy) (bool, error) {
98+
tr := tgPolicy.Spec.TargetRef
99+
var obj client.Object
100+
switch tr.Kind {
101+
case "Service":
102+
obj = &corev1.Service{}
103+
default:
104+
panic("unexpected targetRef Kind=" + tr.Kind)
105+
}
106+
return k8s.ObjExists(ctx, c.client, types.NamespacedName{
107+
Namespace: tgPolicy.Namespace,
108+
Name: string(tr.Name),
109+
}, obj)
110+
}
111+
112+
func (c *TargetGroupPolicyController) updatePolicyCondition(tgPolicy *anv1alpha1.TargetGroupPolicy, reason gwv1alpha2.PolicyConditionReason, msg string) {
113+
status := metav1.ConditionTrue
114+
if reason != gwv1alpha2.PolicyReasonAccepted {
115+
status = metav1.ConditionFalse
116+
}
117+
cnd := metav1.Condition{
118+
Type: string(gwv1alpha2.PolicyConditionAccepted),
119+
Status: status,
120+
Reason: string(reason),
121+
Message: msg,
122+
}
123+
meta.SetStatusCondition(&tgPolicy.Status.Conditions, cnd)
124+
}
125+
126+
func targetGroupPolicyMapFunc(c client.Client, log gwlog.Logger) handler.MapFunc {
127+
return func(ctx context.Context, obj client.Object) []ctrl.Request {
128+
requests := []ctrl.Request{}
129+
policies := &anv1alpha1.TargetGroupPolicyList{}
130+
err := c.List(ctx, policies, &client.ListOptions{Namespace: obj.GetNamespace()})
131+
if err != nil {
132+
log.Error(err)
133+
return requests
134+
}
135+
for _, policy := range policies.Items {
136+
if obj.GetName() == string(policy.Spec.TargetRef.Name) {
137+
requests = append(requests, ctrl.Request{NamespacedName: policy.GetNamespacedName()})
138+
}
139+
}
140+
return requests
141+
}
142+
}

‎helm/crds/application-networking.k8s.aws_targetgrouppolicies.yaml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,14 @@ spec:
157157
- targetRef
158158
type: object
159159
status:
160-
description: TargetGroupPolicyStatus defines the observed state of AccessLogPolicy.
160+
default:
161+
conditions:
162+
- lastTransitionTime: "1970-01-01T00:00:00Z"
163+
message: Waiting for controller
164+
reason: NotReconciled
165+
status: Unknown
166+
type: Accepted
167+
description: Status defines the current state of TargetGroupPolicy.
161168
properties:
162169
conditions:
163170
default:
@@ -254,4 +261,5 @@ spec:
254261
type: object
255262
served: true
256263
storage: true
257-
subresources: {}
264+
subresources:
265+
status: {}

‎pkg/apis/applicationnetworking/v1alpha1/targetgrouppolicy_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,16 @@ const (
2020
// +kubebuilder:resource:categories=gateway-api,shortName=tgp
2121
// +kubebuilder:storageversion
2222
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
23+
// +kubebuilder:subresource:status
2324
type TargetGroupPolicy struct {
2425
metav1.TypeMeta `json:",inline"`
2526
metav1.ObjectMeta `json:"metadata,omitempty"`
2627

2728
Spec TargetGroupPolicySpec `json:"spec"`
2829

30+
// Status defines the current state of TargetGroupPolicy.
31+
//
32+
// +kubebuilder:default={conditions: {{type: "Accepted", status: "Unknown", reason:"NotReconciled", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}}
2933
Status TargetGroupPolicyStatus `json:"status,omitempty"`
3034
}
3135

‎pkg/gateway/model_build_targetgroup.go‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,19 @@ import (
55
"errors"
66
"fmt"
77

8-
apierrors "k8s.io/apimachinery/pkg/api/errors"
9-
10-
"github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog"
11-
8+
"github.com/aws/aws-sdk-go/service/vpclattice"
129
corev1 "k8s.io/api/core/v1"
10+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1311
"k8s.io/apimachinery/pkg/types"
14-
1512
"sigs.k8s.io/controller-runtime/pkg/client"
1613

17-
"github.com/aws/aws-sdk-go/service/vpclattice"
18-
1914
anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1"
2015
"github.com/aws/aws-application-networking-k8s/pkg/config"
2116
"github.com/aws/aws-application-networking-k8s/pkg/k8s"
2217
"github.com/aws/aws-application-networking-k8s/pkg/k8s/policyhelper"
2318
"github.com/aws/aws-application-networking-k8s/pkg/model/core"
2419
model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice"
20+
"github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog"
2521
)
2622

2723
type InvalidBackendRefError struct {
@@ -149,17 +145,16 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetGroup(ctx context.Contex
149145
}
150146
}
151147

152-
tgps, err := policyhelper.GetAttachedPolicies(ctx, t.client, k8s.NamespacedName(t.serviceExport), &anv1alpha1.TargetGroupPolicy{})
148+
tgp, err := policyhelper.GetValidPolicy(ctx, t.client,
149+
k8s.NamespacedName(t.serviceExport), &anv1alpha1.TargetGroupPolicy{})
153150
if err != nil {
154151
return nil, err
155152
}
156153

157154
protocol := "HTTP"
158155
protocolVersion := vpclattice.TargetGroupProtocolVersionHttp1
159156
var healthCheckConfig *vpclattice.HealthCheckConfig
160-
if len(tgps) > 0 {
161-
// TODO: TGP conflicts should be handled correctly w/ status update, for now just picking up one
162-
tgp := tgps[0]
157+
if tgp != nil {
163158
if tgp.Spec.Protocol != nil {
164159
protocol = *tgp.Spec.Protocol
165160
}
@@ -318,17 +313,15 @@ func (t *backendRefTargetGroupModelBuildTask) buildTargetGroupSpec(ctx context.C
318313
}
319314
}
320315

321-
tgps, err := policyhelper.GetAttachedPolicies(ctx, t.client, backendRefNsName, &anv1alpha1.TargetGroupPolicy{})
316+
tgp, err := policyhelper.GetValidPolicy(ctx, t.client, backendRefNsName, &anv1alpha1.TargetGroupPolicy{})
322317
if err != nil {
323318
return model.TargetGroupSpec{}, err
324319
}
325320

326321
protocol := "HTTP"
327322
protocolVersion := vpclattice.TargetGroupProtocolVersionHttp1
328323
var healthCheckConfig *vpclattice.HealthCheckConfig
329-
if len(tgps) > 0 {
330-
// TODO: TGP conflicts should be handled correctly w/ status update, for now just picking up one
331-
tgp := tgps[0]
324+
if tgp != nil {
332325
if tgp.Spec.Protocol != nil {
333326
protocol = *tgp.Spec.Protocol
334327
}

‎pkg/k8s/policyhelper/policy.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ package policyhelper
33
import (
44
"context"
55
"fmt"
6+
"strings"
67

8+
"golang.org/x/exp/slices"
79
corev1 "k8s.io/api/core/v1"
810
"k8s.io/apimachinery/pkg/api/meta"
911
"k8s.io/apimachinery/pkg/types"
@@ -20,6 +22,19 @@ type policyInfo struct {
2022
kind gwv1beta1.Kind
2123
}
2224

25+
func GetValidPolicy[T core.Policy](ctx context.Context, k8sClient client.Client, searchTargetRef types.NamespacedName, policy T) (T, error) {
26+
var empty T
27+
policies, err := GetAttachedPolicies(ctx, k8sClient, searchTargetRef, policy)
28+
conflictResolutionSort(policies)
29+
if err != nil {
30+
return empty, err
31+
}
32+
if len(policies) == 0 {
33+
return empty, nil
34+
}
35+
return policies[0], nil
36+
}
37+
2338
func GetAttachedPolicies[T core.Policy](ctx context.Context, k8sClient client.Client, searchTargetRef types.NamespacedName, policy T) ([]T, error) {
2439
var policies []T
2540
info, err := getPolicyInfo(policy)
@@ -76,3 +91,31 @@ func getPolicyInfo(policyType core.Policy) (policyInfo, error) {
7691
return policyInfo{}, fmt.Errorf("unsupported policy type %T", policyType)
7792
}
7893
}
94+
95+
// sort in-place for policy conflict resolution
96+
// 1. older policy (CreationTimeStamp) has precedence
97+
// 2. alphabetical order namespace, then name
98+
func conflictResolutionSort[T core.Policy](policies []T) {
99+
slices.SortFunc(policies, func(a, b T) int {
100+
tsA := a.GetCreationTimestamp().Time
101+
tsB := b.GetCreationTimestamp().Time
102+
switch {
103+
case tsA.Before(tsB):
104+
return -1
105+
case tsA.After(tsB):
106+
return 1
107+
default:
108+
nsnA := a.GetNamespacedName()
109+
nsnB := b.GetNamespacedName()
110+
nsA := nsnA.Namespace
111+
nsB := nsnB.Namespace
112+
nsCmp := strings.Compare(nsA, nsB)
113+
if nsCmp != 0 {
114+
return nsCmp
115+
}
116+
nA := nsnA.Name
117+
nB := nsnB.Name
118+
return strings.Compare(nA, nB)
119+
}
120+
})
121+
}

0 commit comments

Comments
 (0)