diff --git a/tests/framework/logging.go b/tests/framework/logging.go index ddc02672a3..306fc4b78d 100644 --- a/tests/framework/logging.go +++ b/tests/framework/logging.go @@ -13,7 +13,9 @@ func WithLoggingDisabled() Option { } func LogOptions(opts ...Option) *Options { - options := &Options{logEnabled: true} + options := &Options{ + logEnabled: true, + } for _, opt := range opts { opt(options) } diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 0bb98d1956..e51e32dd49 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -39,12 +39,12 @@ type InstallationConfig struct { // InstallGatewayAPI installs the specified version of the Gateway API resources. func InstallGatewayAPI(apiVersion string) ([]byte, error) { - apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) - GinkgoWriter.Printf("Installing Gateway API version %q at API path %q\n", apiVersion, apiPath) + apiPath := fmt.Sprintf("%s/v%s/experimental-install.yaml", gwInstallBasePath, apiVersion) + GinkgoWriter.Printf("Installing Gateway API CRDs from experimental channel %q", apiVersion, apiPath) cmd := exec.CommandContext( context.Background(), - "kubectl", "apply", "-f", apiPath, + "kubectl", "apply", "--server-side", "--force-conflicts", "-f", apiPath, ) output, err := cmd.CombinedOutput() if err != nil { @@ -59,8 +59,8 @@ func InstallGatewayAPI(apiVersion string) ([]byte, error) { // UninstallGatewayAPI uninstalls the specified version of the Gateway API resources. func UninstallGatewayAPI(apiVersion string) ([]byte, error) { - apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) - GinkgoWriter.Printf("Uninstalling Gateway API version %q at API path %q\n", apiVersion, apiPath) + apiPath := fmt.Sprintf("%s/v%s/experimental-install.yaml", gwInstallBasePath, apiVersion) + GinkgoWriter.Printf("Uninstalling Gateway API CRDs from experimental channel for version %q\n", apiVersion) output, err := exec.CommandContext(context.Background(), "kubectl", "delete", "-f", apiPath).CombinedOutput() if err != nil && !strings.Contains(string(output), "not found") { @@ -84,6 +84,7 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { "--namespace", cfg.Namespace, "--wait", "--set", "nginxGateway.snippetsFilters.enable=true", + "--set", "nginxGateway.gwAPIExperimentalFeatures.enable=true", } if cfg.ChartVersion != "" { args = append(args, "--version", cfg.ChartVersion) diff --git a/tests/framework/prometheus.go b/tests/framework/prometheus.go index 37d72e1c49..f175dec899 100644 --- a/tests/framework/prometheus.go +++ b/tests/framework/prometheus.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "net/http" "os" "os/exec" "time" @@ -542,7 +543,12 @@ func CreateResponseChecker(url, address string, requestTimeout time.Duration, op } return func() error { - status, _, err := Get(url, address, requestTimeout, nil, nil, opts...) + request := Request{ + URL: url, + Address: address, + Timeout: requestTimeout, + } + resp, err := Get(request, opts...) if err != nil { badReqErr := fmt.Errorf("bad response: %w", err) if options.logEnabled { @@ -552,8 +558,8 @@ func CreateResponseChecker(url, address string, requestTimeout time.Duration, op return badReqErr } - if status != 200 { - statusErr := fmt.Errorf("unexpected status code: %d", status) + if resp.StatusCode != http.StatusOK { + statusErr := fmt.Errorf("unexpected status code: %d", resp.StatusCode) if options.logEnabled { GinkgoWriter.Printf("ERROR during creating response checker: %v\n", statusErr) } diff --git a/tests/framework/request.go b/tests/framework/request.go index bd8663b8a3..27308e8e23 100644 --- a/tests/framework/request.go +++ b/tests/framework/request.go @@ -15,18 +15,28 @@ import ( . "github.com/onsi/ginkgo/v2" ) +type Response struct { + Headers http.Header + Body string + StatusCode int +} + +type Request struct { + Body io.Reader + Headers map[string]string + QueryParams map[string]string + URL string + Address string + Timeout time.Duration +} + // Get sends a GET request to the specified url. // It resolves to the specified address instead of using DNS. -// The status and body of the response is returned, or an error. -func Get( - url, address string, - timeout time.Duration, - headers, queryParams map[string]string, - opts ...Option, -) (int, string, error) { +// It returns the response body, headers, and status code. +func Get(request Request, opts ...Option) (Response, error) { options := LogOptions(opts...) - resp, err := makeRequest(http.MethodGet, url, address, nil, timeout, headers, queryParams, opts...) + resp, err := makeRequest(http.MethodGet, request, opts...) if err != nil { if options.logEnabled { GinkgoWriter.Printf( @@ -35,7 +45,7 @@ func Get( ) } - return 0, "", err + return Response{StatusCode: 0}, err } defer resp.Body.Close() @@ -43,24 +53,23 @@ func Get( _, err = body.ReadFrom(resp.Body) if err != nil { GinkgoWriter.Printf("ERROR in Body content: %v returning body: ''\n", err) - return resp.StatusCode, "", err + return Response{StatusCode: resp.StatusCode}, err } if options.logEnabled { GinkgoWriter.Printf("Successfully received response and parsed body: %s\n", body.String()) } - return resp.StatusCode, body.String(), nil + return Response{ + Body: body.String(), + Headers: resp.Header, + StatusCode: resp.StatusCode, + }, nil } // Post sends a POST request to the specified url with the body as the payload. // It resolves to the specified address instead of using DNS. -func Post( - url, address string, - body io.Reader, - timeout time.Duration, - headers, queryParams map[string]string, -) (*http.Response, error) { - response, err := makeRequest(http.MethodPost, url, address, body, timeout, headers, queryParams) +func Post(request Request) (*http.Response, error) { + response, err := makeRequest(http.MethodPost, request) if err != nil { GinkgoWriter.Printf("ERROR occurred during getting response, error: %s\n", err) } @@ -68,13 +77,7 @@ func Post( return response, err } -func makeRequest( - method, url, address string, - body io.Reader, - timeout time.Duration, - headers, queryParams map[string]string, - opts ...Option, -) (*http.Response, error) { +func makeRequest(method string, request Request, opts ...Option) (*http.Response, error) { dialer := &net.Dialer{} transport, ok := http.DefaultTransport.(*http.Transport) @@ -90,10 +93,10 @@ func makeRequest( ) (net.Conn, error) { split := strings.Split(addr, ":") port := split[len(split)-1] - return dialer.DialContext(ctx, network, fmt.Sprintf("%s:%s", address, port)) + return dialer.DialContext(ctx, network, fmt.Sprintf("%s:%s", request.Address, port)) } - ctx, cancel := context.WithTimeout(context.Background(), timeout) + ctx, cancel := context.WithTimeout(context.Background(), request.Timeout) defer cancel() options := LogOptions(opts...) @@ -101,39 +104,41 @@ func makeRequest( requestDetails := fmt.Sprintf( "Method: %s, URL: %s, Address: %s, Headers: %v, QueryParams: %v\n", strings.ToUpper(method), - url, - address, - headers, - queryParams, + request.URL, + request.Address, + request.Headers, + request.QueryParams, ) GinkgoWriter.Printf("Sending request: %s", requestDetails) } - req, err := http.NewRequestWithContext(ctx, method, url, body) + req, err := http.NewRequestWithContext(ctx, method, request.URL, request.Body) if err != nil { return nil, err } - for key, value := range headers { + for key, value := range request.Headers { req.Header.Add(key, value) } - if queryParams != nil { + if request.QueryParams != nil { q := req.URL.Query() - for key, value := range queryParams { + for key, value := range request.QueryParams { q.Add(key, value) } req.URL.RawQuery = q.Encode() } var resp *http.Response - if strings.HasPrefix(url, "https") { + if strings.HasPrefix(request.URL, "https") { // similar to how in our examples with https requests we run our curl command // we turn off verification of the certificate, we do the same here customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // for https test traffic } - client := &http.Client{Transport: customTransport} + client := &http.Client{ + Transport: customTransport, + } resp, err = client.Do(req) if err != nil { return nil, err diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 944a9f0832..9be8ea7af1 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -120,19 +120,28 @@ func expectRequestToRespondFromExpectedServer( headers, queryParams map[string]string, ) error { GinkgoWriter.Printf("Expecting request to respond from the server %q\n", expServerName) - status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout, headers, queryParams) + + request := framework.Request{ + URL: appURL, + Address: address, + Timeout: timeoutConfig.RequestTimeout, + Headers: headers, + QueryParams: queryParams, + } + + resp, err := framework.Get(request) if err != nil { return err } - if status != http.StatusOK { + if resp.StatusCode != http.StatusOK { statusErr := errors.New("http status was not 200") GinkgoWriter.Printf("ERROR: %v\n", statusErr) return statusErr } - actualServerName, err := extractServerName(body) + actualServerName, err := extractServerName(resp.Body) if err != nil { GinkgoWriter.Printf("ERROR extracting server name from response body: %v\n", err) diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index 2422364cf6..5b3e74612b 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -254,7 +254,13 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" _, err := rand.Read(payload) Expect(err).ToNot(HaveOccurred()) - resp, err := framework.Post(url, address, bytes.NewReader(payload), timeoutConfig.RequestTimeout, nil, nil) + request := framework.Request{ + URL: url, + Address: address, + Body: bytes.NewReader(payload), + Timeout: timeoutConfig.RequestTimeout, + } + resp, err := framework.Post(request) Expect(err).ToNot(HaveOccurred()) Expect(resp).To(HaveHTTPStatus(expStatus)) diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index a5d3348958..0b013f7cba 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -555,14 +555,19 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra }) func expectRequestToSucceed(appURL, address string, responseBodyMessage string) error { - status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout, nil, nil) + request := framework.Request{ + URL: appURL, + Address: address, + Timeout: timeoutConfig.RequestTimeout, + } + resp, err := framework.Get(request) - if status != http.StatusOK { - return fmt.Errorf("http status was not 200, got %d: %w", status, err) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http status was not 200, got %d: %w", resp.StatusCode, err) } - if !strings.Contains(body, responseBodyMessage) { - return fmt.Errorf("expected response body to contain correct body message, got: %s", body) + if !strings.Contains(resp.Body, responseBodyMessage) { + return fmt.Errorf("expected response body to contain correct body message, got: %s", resp.Body) } return err @@ -577,13 +582,18 @@ func expectRequestToSucceed(appURL, address string, responseBodyMessage string) // We only want an error returned from this particular function if it does not appear that NGINX has // stopped serving traffic. func expectRequestToFail(appURL, address string) error { - status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout, nil, nil) - if status != 0 { + request := framework.Request{ + URL: appURL, + Address: address, + Timeout: timeoutConfig.RequestTimeout, + } + resp, err := framework.Get(request) + if resp.StatusCode != 0 { return errors.New("expected http status to be 0") } - if body != "" { - return fmt.Errorf("expected response body to be empty, instead received: %s", body) + if resp.Body != "" { + return fmt.Errorf("expected response body to be empty, instead received: %s", resp.Body) } if err == nil { diff --git a/tests/suite/manifests/session-persistence/cafe.yaml b/tests/suite/manifests/session-persistence/cafe.yaml new file mode 100644 index 0000000000..9c1a83548a --- /dev/null +++ b/tests/suite/manifests/session-persistence/cafe.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coffee +spec: + replicas: 3 + selector: + matchLabels: + app: coffee + template: + metadata: + labels: + app: coffee + spec: + containers: + - name: coffee + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: coffee +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: coffee +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tea +spec: + replicas: 3 + selector: + matchLabels: + app: tea + template: + metadata: + labels: + app: tea + spec: + containers: + - name: tea + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: tea +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: tea diff --git a/tests/suite/manifests/session-persistence/gateway.yaml b/tests/suite/manifests/session-persistence/gateway.yaml new file mode 100644 index 0000000000..e6507f613b --- /dev/null +++ b/tests/suite/manifests/session-persistence/gateway.yaml @@ -0,0 +1,11 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway +spec: + gatewayClassName: nginx + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: "*.example.com" diff --git a/tests/suite/manifests/session-persistence/grpc-backends.yaml b/tests/suite/manifests/session-persistence/grpc-backends.yaml new file mode 100644 index 0000000000..fc5011f92b --- /dev/null +++ b/tests/suite/manifests/session-persistence/grpc-backends.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: grpc-backend +spec: + selector: + app: grpc-backend + ports: + - protocol: TCP + port: 8080 + targetPort: 50051 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grpc-backend + labels: + app: grpc-backend +spec: + replicas: 3 + selector: + matchLabels: + app: grpc-backend + template: + metadata: + labels: + app: grpc-backend + spec: + containers: + - name: grpc-backend + image: ghcr.io/nginx/kic-test-grpc-server:0.2.6 + ports: + - containerPort: 50051 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + readinessProbe: + tcpSocket: + port: 50051 + resources: + requests: + cpu: 10m diff --git a/tests/suite/manifests/session-persistence/route-invalid-sp-config.yaml b/tests/suite/manifests/session-persistence/route-invalid-sp-config.yaml new file mode 100644 index 0000000000..0ff8d6481b --- /dev/null +++ b/tests/suite/manifests/session-persistence/route-invalid-sp-config.yaml @@ -0,0 +1,49 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-invalid-sp +spec: + parentRefs: + - name: gateway + sectionName: http + hostnames: + - "cafe.example.com" + rules: + - matches: + - path: + type: Exact + value: / + backendRefs: + - name: tea + port: 80 + sessionPersistence: + sessionName: invalid-cookie + type: Header + idleTimeout: 30m + absoluteTimeout: 10000h # duration too long for NGINX + cookieConfig: + lifetimeType: Session +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpc-route-invalid-sp +spec: + parentRefs: + - name: gateway + sectionName: http + rules: + - matches: + - method: + service: helloworld.Greeter + method: SayHello + backendRefs: + - name: grpc-backend + port: 8080 + sessionPersistence: + sessionName: invalid-cookie + type: Header + idleTimeout: 30m + absoluteTimeout: 10000h # duration too long for NGINX + cookieConfig: + lifetimeType: Session diff --git a/tests/suite/manifests/session-persistence/routes-oss.yaml b/tests/suite/manifests/session-persistence/routes-oss.yaml new file mode 100644 index 0000000000..361542cfe1 --- /dev/null +++ b/tests/suite/manifests/session-persistence/routes-oss.yaml @@ -0,0 +1,35 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: coffee +spec: + parentRefs: + - name: gateway + sectionName: http + hostnames: + - "cafe.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: /coffee + backendRefs: + - name: coffee + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpc-route +spec: + parentRefs: + - name: gateway + sectionName: http + rules: + - matches: + - method: + service: helloworld.Greeter + method: SayHello + backendRefs: + - name: grpc-backend + port: 8080 diff --git a/tests/suite/manifests/session-persistence/routes-plus.yaml b/tests/suite/manifests/session-persistence/routes-plus.yaml new file mode 100644 index 0000000000..d1c370d02f --- /dev/null +++ b/tests/suite/manifests/session-persistence/routes-plus.yaml @@ -0,0 +1,81 @@ +# GRPC Route with method match and unnamed session persistence configuration +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpc-route +spec: + parentRefs: + - name: gateway + sectionName: http + rules: + - matches: + - method: + service: helloworld.Greeter + method: SayHello + backendRefs: + - name: grpc-backend + port: 8080 + sessionPersistence: + type: Cookie + absoluteTimeout: 24h + cookieConfig: + lifetimeType: Permanent +--- +# Route with multiple path matches(common prefix /shop) and unnamed session persistence configuration +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: coffee +spec: + parentRefs: + - name: gateway + hostnames: + - cafe.example.com + rules: + - matches: + - path: + type: PathPrefix + value: /coffee + - path: + type: PathPrefix + value: /coffee/snacks + - path: + type: PathPrefix + value: /coffee/orders/checkout + - path: + type: PathPrefix + value: /coffee/desserts + backendRefs: + - name: coffee + port: 80 + sessionPersistence: + type: Cookie + absoluteTimeout: 48h + cookieConfig: + lifetimeType: Permanent +--- +# Route with regex path match and named session persistence configuration +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: tea +spec: + parentRefs: + - name: gateway + sectionName: http + hostnames: + - "cafe.example.com" + rules: + - matches: + - path: + type: RegularExpression + value: /tea/[a-z]+/flavors + backendRefs: + - name: tea + port: 80 + sessionPersistence: + sessionName: tea-cookie + type: Cookie + absoluteTimeout: 48h + cookieConfig: + lifetimeType: Session diff --git a/tests/suite/manifests/session-persistence/usp.yaml b/tests/suite/manifests/session-persistence/usp.yaml new file mode 100644 index 0000000000..4e7b3a19ae --- /dev/null +++ b/tests/suite/manifests/session-persistence/usp.yaml @@ -0,0 +1,13 @@ +apiVersion: gateway.nginx.org/v1alpha1 +kind: UpstreamSettingsPolicy +metadata: + name: usp-ip-hash +spec: + targetRefs: + - group: core + kind: Service + name: coffee + - group: core + kind: Service + name: grpc-backend + loadBalancingMethod: "ip_hash" diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index f133cc6acd..c5f13ebfff 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -64,16 +64,21 @@ var _ = Describe("Basic test example", Label("functional"), func() { Eventually( func() error { - status, body, err := framework.Get(url, address, timeoutConfig.RequestTimeout, nil, nil) + request := framework.Request{ + URL: url, + Address: address, + Timeout: timeoutConfig.RequestTimeout, + } + resp, err := framework.Get(request) if err != nil { return err } - if status != http.StatusOK { - return fmt.Errorf("status not 200; got %d", status) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status not 200; got %d", resp.StatusCode) } expBody := "URI: /hello" - if !strings.Contains(body, expBody) { - return fmt.Errorf("bad body: got %s; expected %s", body, expBody) + if !strings.Contains(resp.Body, expBody) { + return fmt.Errorf("bad body: got %s; expected %s", resp.Body, expBody) } return nil }). diff --git a/tests/suite/session_persistence_test.go b/tests/suite/session_persistence_test.go new file mode 100644 index 0000000000..4e0433b1d6 --- /dev/null +++ b/tests/suite/session_persistence_test.go @@ -0,0 +1,608 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/nginx/nginx-gateway-fabric/v2/tests/framework" +) + +var invalidSPErrMsgs = "[spec.rules[0].sessionPersistence.type: Unsupported value: \"Header\": " + + "supported values: \"Cookie\", spec.rules[0].sessionPersistence.idleTimeout: " + + "Forbidden: IdleTimeout, spec.rules[0].sessionPersistence.absoluteTimeout: " + + "Invalid value: \"10000h\": duration is too large for NGINX format (exceeds 9999h), " + + "spec.rules[0].sessionPersistence: Invalid value: \"spec.rules[0].sessionPersistence\":" + + " session persistence is ignored because there are errors in the configuration]" + +var _ = Describe("SessionPersistence OSS", Ordered, Label("functional", "session-persistence-oss"), func() { + var ( + files = []string{ + "session-persistence/cafe.yaml", + "session-persistence/grpc-backends.yaml", + "session-persistence/gateway.yaml", + "session-persistence/routes-oss.yaml", + } + + namespace = "session-persistence-oss" + gatewayName = "gateway" + + nginxPodName string + ) + + BeforeAll(func() { + ns := &core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + + Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) + Expect(err).ToNot(HaveOccurred()) + Expect(nginxPodNames).To(HaveLen(1)) + + nginxPodName = nginxPodNames[0] + + setUpPortForward(nginxPodName, namespace) + }) + + AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + cleanUpPortForward() + + Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) + }) + + When("LoadBalancingMethod `ip-hash` is used for session affinity", func() { + uspFiles := []string{ + "session-persistence/usp.yaml", + } + + BeforeAll(func() { + Expect(resourceManager.ApplyFromFiles(uspFiles, namespace)).To(Succeed()) + }) + + AfterAll(func() { + Expect(resourceManager.DeleteFromFiles(uspFiles, namespace)).To(Succeed()) + }) + + Specify("upstreamSettingsPolicies are accepted", func() { + usPolicy := "usp-ip-hash" + + uspolicyNsName := types.NamespacedName{Name: usPolicy, Namespace: namespace} + + err := waitForUSPolicyStatus( + uspolicyNsName, + gatewayName, + metav1.ConditionTrue, + gatewayv1.PolicyReasonAccepted, + ) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("%s was not accepted", usPolicy)) + }) + + Context("verify working traffic", func() { + It("should return 200 response for HTTPRoute `coffee` from the same backend", func() { + port := 80 + if portFwdPort != 0 { + port = portFwdPort + } + baseCoffeeURL := fmt.Sprintf("http://cafe.example.com:%d%s", port, "/coffee") + + Eventually( + func() error { + return expectRequestToSucceedAndRespondFromTheSameBackend(baseCoffeeURL, address, "URI: /coffee", 11) + }). + WithTimeout(timeoutConfig.RequestTimeout). + WithPolling(500 * time.Millisecond). + Should(Succeed()) + }) + }) + + Context("nginx directives", func() { + var conf *framework.Payload + + BeforeAll(func() { + var err error + conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace, "") + Expect(err).ToNot(HaveOccurred()) + }) + + DescribeTable("are set properly for", + func(expCfgs []framework.ExpectedNginxField) { + for _, expCfg := range expCfgs { + Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) + } + }, + Entry("HTTP upstream", []framework.ExpectedNginxField{ + { + Directive: "upstream", + Value: "session-persistence-oss_coffee_80", + File: "http.conf", + }, + { + Directive: "ip_hash", + Upstream: "session-persistence-oss_coffee_80", + File: "http.conf", + }, + }), + Entry("GRPC upstream", []framework.ExpectedNginxField{ + { + Directive: "upstream", + Value: "session-persistence-oss_grpc-backend_8080", + File: "http.conf", + }, + { + Directive: "ip_hash", + Upstream: "session-persistence-oss_grpc-backend_8080", + File: "http.conf", + }, + }), + ) + }) + }) +}) + +var _ = Describe("SessionPersistence Plus", Ordered, Label("functional", "session-persistence-plus"), func() { + var ( + files = []string{ + "session-persistence/cafe.yaml", + "session-persistence/grpc-backends.yaml", + "session-persistence/gateway.yaml", + "session-persistence/routes-plus.yaml", + } + + namespace = "session-persistence-plus" + + nginxPodName string + ) + + BeforeAll(func() { + ns := &core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + + Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) + Expect(err).ToNot(HaveOccurred()) + Expect(nginxPodNames).To(HaveLen(1)) + + nginxPodName = nginxPodNames[0] + setUpPortForward(nginxPodName, namespace) + }) + + AfterAll(func() { + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + cleanUpPortForward() + + Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) + }) + + When("sticky cookies are used for session persistence in NGINX Plus", func() { + var baseCoffeeURL, baseTeaURL string + + BeforeAll(func() { + port := 80 + if portFwdPort != 0 { + port = portFwdPort + } + + baseCoffeeURL = fmt.Sprintf("http://cafe.example.com:%d%s", port, "/coffee") + baseTeaURL = fmt.Sprintf("http://cafe.example.com:%d%s", port, "/tea/location/flavors") + }) + + Context("verify working traffic", func() { + It("should return 200 responses from the same backend for HTTPRoutes `coffee` and `tea`", func() { + if !*plusEnabled { + Skip("Skipping Session Persistence Plus tests on NGINX OSS deployment") + } + Eventually( + func() error { + return expectRequestToSucceedAndReuseCookie(baseCoffeeURL, address, "URI: /coffee", 11) + }). + WithTimeout(timeoutConfig.RequestTimeout). + WithPolling(500 * time.Millisecond). + Should(Succeed()) + + Eventually( + func() error { + return expectRequestToSucceedAndReuseCookie(baseTeaURL, address, "URI: /tea/location/flavors", 11) + }). + WithTimeout(timeoutConfig.RequestTimeout). + WithPolling(500 * time.Millisecond). + Should(Succeed()) + }) + }) + + Context("nginx directives", func() { + var conf *framework.Payload + + BeforeAll(func() { + var err error + conf, err = resourceManager.GetNginxConfig(nginxPodName, namespace, "") + Expect(err).ToNot(HaveOccurred()) + }) + + DescribeTable("are set properly for", + func(expCfgs []framework.ExpectedNginxField) { + if !*plusEnabled { + Skip("Skipping Session Persistence Plus tests on NGINX OSS deployment") + } + for _, expCfg := range expCfgs { + Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) + } + }, + Entry("HTTP upstreams", []framework.ExpectedNginxField{ + { + Directive: "upstream", + Value: "session-persistence-plus_coffee_80_coffee_session-persistence-plus_0", + File: "http.conf", + }, + { + Directive: "sticky", + Value: "cookie sp_coffee_session-persistence-plus_0 expires=48h path=/coffee", + Upstream: "session-persistence-plus_coffee_80_coffee_session-persistence-plus_0", + File: "http.conf", + }, + { + Directive: "state", + Value: "/var/lib/nginx/state/session-persistence-plus_coffee_80.conf", + Upstream: "session-persistence-plus_coffee_80_coffee_session-persistence-plus_0", + File: "http.conf", + }, + { + Directive: "upstream", + Value: "session-persistence-plus_tea_80_tea_session-persistence-plus_0", + File: "http.conf", + }, + { + Directive: "sticky", + Value: "cookie tea-cookie", + Upstream: "session-persistence-plus_tea_80_tea_session-persistence-plus_0", + File: "http.conf", + }, + { + Directive: "state", + Value: "/var/lib/nginx/state/session-persistence-plus_tea_80.conf", + Upstream: "session-persistence-plus_tea_80_tea_session-persistence-plus_0", + File: "http.conf", + }, + }), + Entry("GRPC upstream", []framework.ExpectedNginxField{ + { + Directive: "upstream", + Value: "session-persistence-plus_grpc-backend_8080_grpc-route_session-persistence-plus_0", + File: "http.conf", + }, + { + Directive: "sticky", + Value: "cookie sp_grpc-route_session-persistence-plus_0 expires=24h", + Upstream: "session-persistence-plus_grpc-backend_8080_grpc-route_session-persistence-plus_0", + File: "http.conf", + }, + { + Directive: "state", + Upstream: "session-persistence-plus_grpc-backend_8080_grpc-route_session-persistence-plus_0", + Value: "/var/lib/nginx/state/session-persistence-plus_grpc-backend_8080.conf", + File: "http.conf", + }, + }), + ) + }) + }) + + When("Routes have an invalid session persistence configuration", func() { + BeforeAll(func() { + routeFile := "session-persistence/route-invalid-sp-config.yaml" + Expect(resourceManager.ApplyFromFiles([]string{routeFile}, namespace)).To(Succeed()) + }) + + It("updates the HTTPRoute status with all relevant validation errors", func() { + if !*plusEnabled { + Skip("Skipping Session Persistence Plus tests on NGINX OSS deployment") + } + routeNsName := types.NamespacedName{Name: "route-invalid-sp", Namespace: namespace} + err := waitForHTTPRouteToHaveErrorMessage(routeNsName) + Expect(err).ToNot(HaveOccurred(), "expected route to report invalid session persistence configuration") + }) + + It("updates the HTTPRoute status with all relevant validation errors", func() { + if !*plusEnabled { + Skip("Skipping Session Persistence Plus tests on NGINX OSS deployment") + } + routeNsName := types.NamespacedName{Name: "grpc-route-invalid-sp", Namespace: namespace} + err := waitForGRPCRouteToHaveErrorMessage(routeNsName) + Expect(err).ToNot(HaveOccurred(), "expected route to report invalid session persistence configuration") + }) + }) +}) + +func waitForHTTPRouteToHaveErrorMessage(routeNsName types.NamespacedName) error { + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetStatusTimeout) + defer cancel() + + GinkgoWriter.Printf( + "Waiting for %q to have the condition Accepted/True/Accepted with the right error message\n", + routeNsName, + ) + + return wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + var route gatewayv1.HTTPRoute + if err := resourceManager.Get(ctx, routeNsName, &route); err != nil { + return false, err + } + + return checkRouteStatus( + route.Status.RouteStatus, + gatewayv1.RouteConditionAccepted, + metav1.ConditionTrue, + invalidSPErrMsgs, + ) + }, + ) +} + +func waitForGRPCRouteToHaveErrorMessage(routeNsName types.NamespacedName) error { + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetStatusTimeout) + defer cancel() + + GinkgoWriter.Printf( + "Waiting for %q to have the condition Accepted/True/Accepted with the right error message\n", + routeNsName, + ) + + return wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + var route gatewayv1.GRPCRoute + if err := resourceManager.Get(ctx, routeNsName, &route); err != nil { + return false, err + } + + return checkRouteStatus( + route.Status.RouteStatus, + gatewayv1.RouteConditionAccepted, + metav1.ConditionTrue, + invalidSPErrMsgs) + }, + ) +} + +func checkRouteStatus( + rs gatewayv1.RouteStatus, + conditionType gatewayv1.RouteConditionType, + condStatus metav1.ConditionStatus, + expectedReasonSubstring string, +) (bool, error) { + var err error + if len(rs.Parents) == 0 { + GinkgoWriter.Printf("route does not have a status yet\n") + return false, nil + } + if len(rs.Parents) != 1 { + err := fmt.Errorf("route has %d parents, expected 1", len(rs.Parents)) + GinkgoWriter.Printf("ERROR: %v\n", err) + return false, err + } + + parent := rs.Parents[0] + if parent.Conditions == nil { + err := fmt.Errorf("route has no conditions in its status") + GinkgoWriter.Printf("ERROR: %v\n", err) + return false, err + } + if len(parent.Conditions) != 2 { + err := fmt.Errorf("expected route to have only two conditions, instead has %d", len(parent.Conditions)) + GinkgoWriter.Printf("ERROR: %v\n", err) + return false, err + } + + cond := parent.Conditions[1] + if cond.Type != string(conditionType) && + cond.Status != condStatus && + !strings.Contains(cond.Reason, expectedReasonSubstring) { + err := fmt.Errorf( + "expected route condition to be Type=%s, Status=%s, "+ + "Reason contains=%s; instead got Type=%s, Status=%s, Reason=%s", + conditionType, condStatus, expectedReasonSubstring, cond.Type, cond.Status, cond.Reason, + ) + GinkgoWriter.Printf("ERROR: %v\n", err) + return false, err + } + + return err == nil, nil +} + +func expectRequestToSucceedAndRespondFromTheSameBackend( + appURL, + address, + responseBodyMessage string, + totalRequests int, +) error { + var firstServerName string + + for i := range totalRequests { + request := framework.Request{ + URL: appURL, + Address: address, + Timeout: timeoutConfig.RequestTimeout, + } + resp, err := framework.Get(request) + if err != nil { + return fmt.Errorf("request %d to %s failed: %w", i+1, appURL, err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("request %d: http status was not 200, got %d", i+1, resp.StatusCode) + } + + if !strings.Contains(resp.Body, responseBodyMessage) { + return fmt.Errorf("request %d: expected response body to contain %q, got: %s", i+1, responseBodyMessage, resp.Body) + } + + serverName, err := extractServerName(resp.Body) + if err != nil { + return fmt.Errorf("request %d: failed to extract server name: %w; body: %s", i+1, err, resp.Body) + } + + if i == 0 { + firstServerName = serverName + continue + } + + // subsequent replies must come from the same backend. + if serverName != firstServerName { + return fmt.Errorf( + "request %d: expected server name %q, got %q resulting in `ip-hash` stickiness failure", + i+1, firstServerName, serverName, + ) + } + } + + return nil +} + +func expectRequestToSucceedAndReuseCookie( + appURL, + address, + responseBodyMessage string, + totalRequests int, +) error { + var firstServerName string + cookieAttr := make(map[string]string, 0) + + for i := range totalRequests { + headers := make(map[string]string, 0) + + // send cookie token after first response + if i > 0 { + if cookieAttr == nil { + return fmt.Errorf("request %d: cookie attributes are nil after first response", i+1) + } + + headers["Cookie"] = fmt.Sprintf("%s=%s", cookieAttr["name"], cookieAttr["value"]) + } + + request := framework.Request{ + URL: appURL, + Address: address, + Timeout: timeoutConfig.RequestTimeout, + Headers: headers, + } + + resp, err := framework.Get(request) + if err != nil { + return fmt.Errorf("request %d to %s failed: %w", i+1, appURL, err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("request %d: http status was not 200, got %d", i+1, resp.StatusCode) + } + + if !strings.Contains(resp.Body, responseBodyMessage) { + return fmt.Errorf( + "request %d: expected response body to contain %q, got: %s", + i+1, responseBodyMessage, resp.Body, + ) + } + + serverName, err := extractServerName(resp.Body) + if err != nil { + return fmt.Errorf( + "request %d: failed to extract server name: %w; body: %s", + i+1, err, resp.Body, + ) + } + + // get the cookie token from the first response + if i == 0 { + cookieAttr, err = extractCookieInformationFromResponseHeaders(resp.Headers) + if err != nil { + return fmt.Errorf( + "request %d: failed to extract cookie from response headers: %w; body: %s", + i+1, err, resp.Body, + ) + } + + firstServerName = serverName + continue + } + + if serverName != firstServerName { + return fmt.Errorf( + "request %d: expected server name %q, got %q (session persistence failed)", + i+1, firstServerName, serverName, + ) + } + } + + return nil +} + +func extractCookieInformationFromResponseHeaders(h http.Header) (map[string]string, error) { + values := h.Values("Set-Cookie") + if len(values) == 0 { + return nil, fmt.Errorf("no Set-Cookie header found in response") + } + + raw := strings.TrimSpace(values[0]) + if raw == "" { + return nil, fmt.Errorf("empty Set-Cookie header") + } + + parts := strings.Split(raw, ";") + if len(parts) == 0 { + return nil, fmt.Errorf("malformed Set-Cookie header: %q", raw) + } + + // first part is cookie-name=value + pair := strings.TrimSpace(parts[0]) + nv := strings.SplitN(pair, "=", 2) + if len(nv) != 2 { + return nil, fmt.Errorf("malformed Set-Cookie header (no name=value): %q", raw) + } + + name := strings.TrimSpace(nv[0]) + value := strings.TrimSpace(nv[1]) + if name == "" || value == "" { + return nil, fmt.Errorf("malformed Set-Cookie header (empty name or value): %q", raw) + } + + result := map[string]string{ + "name": name, + "value": value, + } + + return result, nil +} diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go index 1df06598df..b52012fc17 100644 --- a/tests/suite/tracing_test.go +++ b/tests/suite/tracing_test.go @@ -133,19 +133,17 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac for range count { Eventually( func() error { - status, _, err := framework.Get( - url, - address, - timeoutConfig.RequestTimeout, - nil, - nil, - framework.WithLoggingDisabled(), - ) + request := framework.Request{ + URL: url, + Address: address, + Timeout: timeoutConfig.RequestTimeout, + } + resp, err := framework.Get(request, framework.WithLoggingDisabled()) if err != nil { return err } - if status != http.StatusOK { - return fmt.Errorf("status not 200; got %d", status) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status not 200; got %d", resp.StatusCode) } return nil }).