Skip to content

Commit 553422e

Browse files
authored
Update Helm chart to include pod mutating webhook for readiness gates (#612)
Update Helm chart to include pod mutating webhook for readiness gates. Moved webhook certificate provisioning out of build. Added helper scripts and env variable for enabling webhooks and working with certificates
1 parent 6e262ae commit 553422e

File tree

14 files changed

+286
-62
lines changed

14 files changed

+286
-62
lines changed

.github/workflows/e2e-test.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ jobs:
8181
- name: Run test
8282
run: |
8383
make e2e-test
84+
make webhook-e2e-test
8485
- name: Cleanup
8586
if: always()
8687
run: |

Makefile

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -101,16 +101,8 @@ docker-push: ## Push docker image with the manager.
101101
# also generates a placeholder cert for the webhook - this cert is not intended to be valid
102102
.PHONY: build-deploy
103103
build-deploy: ## Create a deployment file that can be applied with `kubectl apply -f deploy.yaml`
104-
$(eval TEMP_KEY := $(shell mktemp))
105-
$(eval TEMP_CERT := $(shell mktemp))
106104
cd config/manager && kustomize edit set image controller=${ECRIMAGES}
107105
kustomize build config/default > deploy.yaml
108-
openssl req -x509 -nodes -days 1 -newkey rsa:2048 -keyout $(TEMP_KEY) -out $(TEMP_CERT) -subj "/CN=not-a-real-cn/O=not-a-real-o" > /dev/null 2>&1
109-
export KEY_B64=`cat $(TEMP_KEY) | base64` && \
110-
export CERT_B64=`cat $(TEMP_CERT) | base64` && \
111-
yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.crt") = env(CERT_B64)' deploy.yaml && \
112-
yq -i e '(.[] as $$item | select(.metadata.name == "webhook-cert" and .kind == "Secret") | .data."tls.key") = env(KEY_B64)' deploy.yaml 2>&1
113-
rm $(TEMP_KEY) $(TEMP_CERT)
114106

115107
.PHONY: manifest
116108
manifest: ## Generate CRD manifest
@@ -155,11 +147,8 @@ docs:
155147
mkdocs build
156148

157149
# NB webhook tests can only run if the controller is deployed to the cluster
158-
webhook-e2e-test-namespace := "webhook-e2e-test"
159-
160150
.PHONY: webhook-e2e-test
161151
webhook-e2e-test:
162-
@kubectl create namespace $(webhook-e2e-test-namespace) > /dev/null 2>&1 || true # ignore already exists error
163152
LOG_LEVEL=debug
164153
cd test && go test \
165154
-p 1 \

cmd/aws-application-networking-k8s/main.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"go.uber.org/zap/zapcore"
2424
"os"
2525
k8swebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
26+
"strings"
2627

2728
"github.com/aws/aws-application-networking-k8s/pkg/aws"
2829
"github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog"
@@ -132,14 +133,17 @@ func main() {
132133

133134
// do not create the webhook server when running locally
134135
var webhookServer k8swebhook.Server
135-
isLocalDev := config.DevMode != ""
136-
if !isLocalDev {
136+
enableWebhook := strings.ToLower(config.WebhookEnabled) == "true"
137+
if enableWebhook {
138+
setupLog.Info("Webhook is enabled, 'webhook-cert' secret must contain a valid TLS key and cert")
137139
webhookServer = k8swebhook.NewServer(k8swebhook.Options{
138140
Port: 9443,
139141
CertDir: "/etc/webhook-cert/",
140142
CertName: "tls.crt",
141143
KeyName: "tls.key",
142144
})
145+
} else {
146+
setupLog.Infof("Webhook is disabled, value: '%s'", config.WebhookEnabled)
143147
}
144148

145149
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -156,8 +160,7 @@ func main() {
156160
setupLog.Fatal("manager setup failed:", err)
157161
}
158162

159-
if !isLocalDev {
160-
// register webhook handlers
163+
if enableWebhook {
161164
readinessGateInjector := webhook.NewPodReadinessGateInjector(
162165
mgr.GetClient(),
163166
log.Named("pod-readiness-gate-injector"),

config/manager/manager.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ spec:
5151
port: 8081
5252
initialDelaySeconds: 5
5353
periodSeconds: 10
54+
env:
55+
- name: WEBHOOK_ENABLED
56+
value: ""
5457
# TODO(user): Configure the resources accordingly based on the project requirements.
5558
# More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
5659
resources:

config/webhook/manifests.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
apiVersion: admissionregistration.k8s.io/v1
33
kind: MutatingWebhookConfiguration
44
metadata:
5-
creationTimestamp: null
65
name: aws-appnet-gwc-mutating-webhook
76
webhooks:
87
- admissionReviewVersions:

docs/guides/environment.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,17 @@ Default: ""
7575
When set as "true", the controller will run in "single service network" mode that will override all gateways
7676
to point to default service network, instead of searching for service network with the same name.
7777
Can be used for small setups and conformance tests.
78+
79+
---
80+
81+
#### `WEBHOOK_ENABLED`
82+
83+
Type: string
84+
85+
Default: ""
86+
87+
When set as "true", the controller will start the webhook listener responsible for pod readiness gate injection
88+
(see ```pod-readiness-gates.md```). This is disabled by default for ```deploy.yaml``` because the controller will not start
89+
successfully without the TLS certificate for the webhook in place. While this can be fixed by running
90+
```scripts/gen-webhook-cert.sh```, it requires manual action. The webhook is enabled by default for the Helm install
91+
as the Helm install will also generate the necessary certificate.

docs/guides/pod-readiness-gates.md

Lines changed: 36 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -18,53 +18,45 @@ This prevents the rolling update of a deployment from terminating old pods until
1818
## Setup
1919
Pod readiness gates rely on [»admission webhooks«](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), where the Kubernetes API server makes calls to the AWS Gateway API controller as part of pod creation. This call is made using TLS, so the controller must present a TLS certificate. This certificate is stored as a standard Kubernetes secret. If you are using Helm, the certificate will automatically be configured as part of the Helm install.
2020

21-
If you are manually deploying the controller, for example using the ```deploy.yaml``` file, you will need to create the tls secret for the webhook in the controller namespace. The ```deploy.yaml``` file includes a placeholder secret, but it must be updated if you wish to use the webhook. The placeholder secret _will not_ pass API server validations, but will ensure the controller container is able to start.
21+
If you are manually deploying the controller using the ```deploy.yaml``` file, you will need to either patch the ```deploy.yaml``` file (see ```scripts/patch-deploy-yaml.sh```) or generate the secret following installation (see ```scripts/gen-webhook-secret.sh```) and manually enable the webhook via the ```WEBHOOK_ENABLED``` environment variable.
2222

23-
### Webhook secret requirements
24-
The webhook requires a specific kubernetes secret to exist in the same namespace as the webhook itself:
25-
* secret name: ```webhook-cert```
26-
* default controller namespace: ```aws-application-networking-system```
27-
```console
28-
# example create-secret command, assumes tls.crt and tls.key exist in current directory
29-
# if the placeholder secret exists, you will need to delete it before setting the new value
30-
kubectl create secret tls webhook-cert --namespace aws-application-networking-system --cert=tls.crt --key=tls.key
23+
Note that, without the secret in place, the controller cannot start successfully, and you will see an error message like the following:
3124
```
32-
33-
### Webhook secret configuration example
34-
The below example creates an unsigned certificate, adds it as the webhook secret, then patches the webhook configuration so the API server trusts the certificate.
35-
36-
If your cluster uses its own PKI and includes appropriate trust configuration for the API server, the certificate issued would be signed by your internal certificate authority and therefore not require the ```kubectl patch``` command below.
37-
```console
38-
# Example commands to configure the webhook to use an unsigned certificate
39-
CERT_FILE=tls.crt
40-
KEY_FILE=tls.key
41-
42-
WEBHOOK_SVC_NAME=webhook-service
43-
WEBHOOK_NAME=aws-appnet-gwc-mutating-webhook
44-
WEBHOOK_NAMESPACE=aws-application-networking-system
45-
WEBHOOK_SECRET_NAME=webhook-cert
46-
47-
# Step 1: generate a certificate if needed, can also be provisioned through orgnanizational PKI, etc
48-
# This cert includes a 100 year expiry
49-
HOST=${WEBHOOK_SVC_NAME}.${WEBHOOK_NAMESPACE}.svc
50-
openssl req -x509 -nodes -days 36500 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}" \
51-
-addext "subjectAltName = DNS:${HOST}, DNS:${HOST}.cluster.local"
52-
53-
# Step 2: replace the placeholder secret from deploy.yaml
54-
kubectl delete secret $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE
55-
kubectl create secret tls $WEBHOOK_SECRET_NAME --namespace $WEBHOOK_NAMESPACE --cert=${CERT_FILE} --key=${KEY_FILE}
56-
57-
# Step 3: Patch the webhook CA bundle to exactly the cert being used.
58-
# This will ensure Kubernetes API server is able to trust the certificate presented by the webhook.
59-
# This step would not be required if you are using a signed certificate that is already trusted by the API server
60-
CERT_B64=$(cat tls.crt | base64)
61-
kubectl patch mutatingwebhookconfigurations.admissionregistration.k8s.io $WEBHOOK_NAME \
62-
--namespace $WEBHOOK_NAMESPACE --type='json' \
63-
-p="[{'op': 'replace', 'path': '/webhooks/0/clientConfig/caBundle', 'value': '${CERT_B64}'}]"
25+
{"level":"error","ts":"...","logger":"setup","caller":"workspace/main.go:240","msg":"tls: failed to find any PEM data in certificate inputproblem running manager"}
26+
```
27+
For this reason, the webhook is ```DISABLED``` by default in the controller for the non-Helm install. You can enable the webhook by setting the ```WEBHOOK_ENABLED``` environment variable to "true" in the ```deploy.yaml``` file.
28+
```yaml
29+
apiVersion: apps/v1
30+
kind: Deployment
31+
metadata:
32+
name: gateway-api-controller
33+
namespace: aws-application-networking-system
34+
labels:
35+
control-plane: gateway-api-controller
36+
spec:
37+
...
38+
template:
39+
metadata:
40+
annotations:
41+
kubectl.kubernetes.io/default-container: manager
42+
labels:
43+
control-plane: gateway-api-controller
44+
spec:
45+
securityContext:
46+
runAsNonRoot: true
47+
containers:
48+
- command:
49+
...
50+
name: manager
51+
...
52+
env:
53+
- name: WEBHOOK_ENABLED
54+
value: "true" # <-- value of "true" enables the webhook in the controller
6455
```
56+
If you run ```scripts/patch-deploy-yaml.sh``` prior to installing ```deploy.yaml```, the script will create the necessary TLS certificates and configuration and will enable the webhook in the controller. Note that, even with the webhook enabled, the webhook will only run for namespaces labeled with `application-networking.k8s.aws/pod-readiness-gate-inject: enabled`.
6557

66-
## Configuration
67-
Pod readiness gate support is enabled by default on the AWS Gateway API controller. To enable the feature, you must apply a label to each of the namespaces you would like to use this feature. You can create and label a namespace as follows -
58+
## Enabling the readiness gate
59+
After a Helm install or manually configuring and enabling the webhook, you are ready to begin using pod readiness gates. Apply a label to each namespace you would like to use this feature. You can create and label a namespace as follows -
6860

6961
```
7062
$ kubectl create namespace example-ns
@@ -81,7 +73,7 @@ Annotations: <none>
8173
Status: Active
8274
```
8375
84-
Once labelled, the controller will add the pod readiness gates to all subsequently created pods.
76+
Once labelled, the controller will add the pod readiness gates to all subsequently created pods in the namespace.
8577
8678
The readiness gates have the condition type ```application-networking.k8s.aws/pod-readiness-gate``` and the controller injects the config to the pod spec only during pod creation.
8779

helm/templates/_helpers.tpl

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,21 @@ If release name contains chart name it will be used as a full name.
3030
{{- define "service-account.name" -}}
3131
{{ default "default" .Values.serviceAccount.name }}
3232
{{- end -}}
33+
34+
{{/* Import or generate certificates for webhook */}}
35+
{{- define "aws-gateway-controller.webhookTLS" -}}
36+
{{- if (and .Values.webhookTLS.caCert .Values.webhookTLS.cert .Values.webhookTLS.key) -}}
37+
caCert: {{ .Values.webhookTLS.caCert }}
38+
cert: {{ .Values.webhookTLS.cert }}
39+
key: {{ .Values.webhookTLS.key }}
40+
{{- else -}}
41+
{{- $ca := genCA "aws-gateway-controller-ca" 36500 -}}
42+
{{- $serviceDefaultName:= printf "webhook-service.%s.svc" .Release.Namespace -}}
43+
{{- $secretName := "webhook-cert" -}}
44+
{{- $altNames := list ($serviceDefaultName) (printf "%s.cluster.local" $serviceDefaultName) -}}
45+
{{- $cert := genSignedCert $serviceDefaultName nil $altNames 36500 $ca -}}
46+
caCert: {{ $ca.Cert | b64enc }}
47+
cert: {{ $cert.Cert | b64enc }}
48+
key: {{ $cert.Key | b64enc }}
49+
{{- end -}}
50+
{{- end -}}

helm/templates/deployment.yaml

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ spec:
5252
ports:
5353
- name: http
5454
containerPort: {{ .Values.deployment.containerPort }}
55+
- name: webhook-server
56+
containerPort: 9443
5557
resources:
5658
{{- toYaml .Values.resources | nindent 10 }}
5759
livenessProbe:
@@ -72,6 +74,10 @@ spec:
7274
drop:
7375
- ALL
7476
readOnlyRootFilesystem: true
77+
volumeMounts:
78+
- mountPath: /etc/webhook-cert
79+
name: webhook-cert
80+
readOnly: true
7581
env:
7682
- name: REGION
7783
value: {{ .Values.awsRegion | quote }}
@@ -87,7 +93,14 @@ spec:
8793
value: {{ .Values.defaultServiceNetwork | quote }}
8894
- name: LOG_LEVEL
8995
value: {{ .Values.log.level | quote }}
96+
- name: WEBHOOK_ENABLED
97+
value: {{ .Values.webhookEnabled | quote }}
9098
terminationGracePeriodSeconds: 10
99+
volumes:
100+
- name: webhook-cert
101+
secret:
102+
defaultMode: 420
103+
secretName: webhook-cert
91104
nodeSelector: {{ toYaml .Values.deployment.nodeSelector | nindent 8 }}
92105
{{ if .Values.deployment.tolerations -}}
93106
tolerations: {{ toYaml .Values.deployment.tolerations | nindent 8 }}
@@ -97,4 +110,4 @@ spec:
97110
{{ end -}}
98111
{{ if .Values.deployment.priorityClassName -}}
99112
priorityClassName: {{ .Values.deployment.priorityClassName }}
100-
{{ end -}}
113+
{{ end -}}

helm/templates/webhook.yaml

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
{{ $tls := fromYaml ( include "aws-gateway-controller.webhookTLS" . ) }}
2+
---
3+
apiVersion: admissionregistration.k8s.io/v1
4+
kind: MutatingWebhookConfiguration
5+
metadata:
6+
name: aws-appnet-gwc-mutating-webhook
7+
webhooks:
8+
- admissionReviewVersions:
9+
- v1
10+
clientConfig:
11+
caBundle: {{ $tls.caCert }}
12+
service:
13+
name: webhook-service
14+
namespace: {{ .Release.Namespace }}
15+
path: /mutate-pod
16+
failurePolicy: Fail
17+
name: mpod.gwc.k8s.aws
18+
rules:
19+
- apiGroups:
20+
- ""
21+
apiVersions:
22+
- v1
23+
operations:
24+
- CREATE
25+
resources:
26+
- pods
27+
sideEffects: None
28+
namespaceSelector:
29+
matchExpressions:
30+
- key: application-networking.k8s.aws/pod-readiness-gate-inject
31+
operator: In
32+
values:
33+
- enabled
34+
objectSelector:
35+
matchExpressions:
36+
- key: app.kubernetes.io/name
37+
operator: NotIn
38+
values:
39+
- gateway-api-controller
40+
---
41+
apiVersion: v1
42+
kind: Service
43+
metadata:
44+
name: webhook-service
45+
namespace: {{ .Release.Namespace }}
46+
spec:
47+
ports:
48+
- port: 443
49+
targetPort: webhook-server
50+
selector:
51+
control-plane: gateway-api-controller
52+
---
53+
apiVersion: v1
54+
kind: Secret
55+
metadata:
56+
name: webhook-cert
57+
namespace: {{ .Release.Namespace }}
58+
type: kubernetes.io/tls
59+
data:
60+
ca.crt: {{ $tls.caCert }}
61+
tls.crt: {{ $tls.cert }}
62+
tls.key: {{ $tls.key }}

0 commit comments

Comments
 (0)