test(mcp): use envtest built-in features to apply CRDs (#516)

Trying to speed up the test cases that require additional
kubernetes APIs.

Each time a test suite or test case requires an additional
API (like OpenShift Routes or Kiali CRDs), provided by
k8s.io/apiextensions, a fixed 2 seconds delay is introduced.

There's no way to reduce that delay by using envtest features
or APIServer CLI flags.

The delays are introduced by kube-aggregator, apiextensions, and others:

https://github.com/kubernetes/kube-aggregator/blob/a00232cf7e758c2771b3542033bec19e69e6451a/pkg/controllers/openapiv3/controller.go#L34

https://github.com/kubernetes/apiextensions-apiserver/blob/8db5ab628dd026827c1c9677944432db70c065c3/pkg/apiserver/customresource_handler.go#L381-L394

To mitigate this, we're going to declare all CRDs used in the test in
the common_test.go TestMain function.

This will make sure we only have a single delay at the beginning of
the entire test suite instead of per test suite or test case.

The CRDs are NOT served by default.

For each test suite we'll be able to enable them or disable them
by using the EnvTestEnableCRD and EnvTestDisableCRD functions.
These will also ensure that the API server lists the new APIs or not.

Signed-off-by: Marc Nuri <marc@marcnuri.com>

Author: manusa

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/google/jsonschema-go v0.3.0
 	github.com/mark3labs/mcp-go v0.43.1
 	github.com/modelcontextprotocol/go-sdk v1.1.0
-	github.com/pkg/errors v0.9.1
 	github.com/spf13/afero v1.15.0
 	github.com/spf13/cobra v1.10.1
 	github.com/spf13/pflag v1.0.10
@@ -104,6 +103,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.22.0 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect

pkg/mcp/common_crd_test.go

@@ -0,0 +1,137 @@
+package mcp
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	apiextensionsv1spec "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/discovery"
+	"k8s.io/utils/ptr"
+)
+
+func CRD(group, version, resource, kind, singular string, namespaced bool) *apiextensionsv1spec.CustomResourceDefinition {
+	scope := "Cluster"
+	if namespaced {
+		scope = "Namespaced"
+	}
+	crd := &apiextensionsv1spec.CustomResourceDefinition{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: apiextensionsv1spec.SchemeGroupVersion.String(),
+			Kind:       "CustomResourceDefinition",
+		},
+		ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s.%s", resource, group)},
+		Spec: apiextensionsv1spec.CustomResourceDefinitionSpec{
+			Group: group,
+			Versions: []apiextensionsv1spec.CustomResourceDefinitionVersion{
+				{
+					Name:    version,
+					Served:  false,
+					Storage: true,
+					Schema: &apiextensionsv1spec.CustomResourceValidation{
+						OpenAPIV3Schema: &apiextensionsv1spec.JSONSchemaProps{
+							Type:                   "object",
+							XPreserveUnknownFields: ptr.To(true),
+						},
+					},
+				},
+			},
+			Scope: apiextensionsv1spec.ResourceScope(scope),
+			Names: apiextensionsv1spec.CustomResourceDefinitionNames{
+				Plural:     resource,
+				Singular:   singular,
+				Kind:       kind,
+				ShortNames: []string{singular},
+			},
+		},
+	}
+	return crd
+}
+
+func EnvTestEnableCRD(ctx context.Context, group, version, resource string) error {
+	apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig)
+	_, err := apiExtensionsV1Client.CustomResourceDefinitions().Patch(
+		ctx,
+		fmt.Sprintf("%s.%s", resource, group),
+		types.JSONPatchType,
+		[]byte(`[{"op": "replace", "path": "/spec/versions/0/served", "value": true}]`),
+		metav1.PatchOptions{})
+	if err != nil {
+		return err
+	}
+	return EnvTestWaitForAPIResourceCondition(ctx, group, version, resource, true)
+}
+
+func EnvTestDisableCRD(ctx context.Context, group, version, resource string) error {
+	apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig)
+	_, err := apiExtensionsV1Client.CustomResourceDefinitions().Patch(
+		ctx,
+		fmt.Sprintf("%s.%s", resource, group),
+		types.JSONPatchType,
+		[]byte(`[{"op": "replace", "path": "/spec/versions/0/served", "value": false}]`),
+		metav1.PatchOptions{})
+	if err != nil {
+		return err
+	}
+	return EnvTestWaitForAPIResourceCondition(ctx, group, version, resource, false)
+}
+
+func EnvTestWaitForAPIResourceCondition(ctx context.Context, group, version, resource string, shouldBeAvailable bool) error {
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(envTestRestConfig)
+	if err != nil {
+		return fmt.Errorf("failed to create discovery client: %w", err)
+	}
+
+	groupVersion := fmt.Sprintf("%s/%s", group, version)
+	if group == "" {
+		groupVersion = version
+	}
+
+	return wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (bool, error) {
+		resourceList, err := discoveryClient.ServerResourcesForGroupVersion(groupVersion)
+		if err != nil {
+			// If we're waiting for the resource to be unavailable and we get an error, it might be gone
+			if !shouldBeAvailable {
+				return true, nil
+			}
+			// Otherwise, keep polling
+			return false, nil
+		}
+
+		// Check if the resource exists in the list
+		found := false
+		for _, apiResource := range resourceList.APIResources {
+			if apiResource.Name == resource {
+				found = true
+				break
+			}
+		}
+
+		// Return true if the condition is met
+		if shouldBeAvailable {
+			return found, nil
+		}
+		return !found, nil
+	})
+}
+
+// EnvTestInOpenShift sets up the kubernetes environment to seem to be running OpenShift
+func EnvTestInOpenShift(ctx context.Context) error {
+	tasks, _ := errgroup.WithContext(ctx)
+	tasks.Go(func() error { return EnvTestEnableCRD(ctx, "project.openshift.io", "v1", "projects") })
+	tasks.Go(func() error { return EnvTestEnableCRD(ctx, "route.openshift.io", "v1", "routes") })
+	return tasks.Wait()
+}
+
+// EnvTestInOpenShiftClear clears the kubernetes environment so it no longer seems to be running OpenShift
+func EnvTestInOpenShiftClear(ctx context.Context) error {
+	tasks, _ := errgroup.WithContext(ctx)
+	tasks.Go(func() error { return EnvTestDisableCRD(ctx, "project.openshift.io", "v1", "projects") })
+	tasks.Go(func() error { return EnvTestDisableCRD(ctx, "route.openshift.io", "v1", "routes") })
+	return tasks.Wait()
+}

pkg/mcp/common_test.go

@@ -2,29 +2,20 @@ package mcp
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
-	"time"
 
 	"github.com/mark3labs/mcp-go/client/transport"
-	"github.com/pkg/errors"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/suite"
-	"golang.org/x/sync/errgroup"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1spec "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	toolswatch "k8s.io/client-go/tools/watch"
-	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/tools/setup-envtest/env"
 	"sigs.k8s.io/controller-runtime/tools/setup-envtest/remote"
@@ -49,6 +40,8 @@ func TestMain(m *testing.M) {
 	// Set high rate limits to avoid client-side throttling in tests
 	_ = os.Setenv("KUBE_CLIENT_QPS", "1000")
 	_ = os.Setenv("KUBE_CLIENT_BURST", "2000")
+	//// Enable control plane output to see API server logs
+	//_ = os.Setenv("KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT", "true")
 	envTestDir, err := store.DefaultStoreDir()
 	if err != nil {
 		panic(err)
@@ -73,7 +66,21 @@ func TestMain(m *testing.M) {
 	versionDir := envTestEnv.Platform.BaseName(*envTestEnv.Version.AsConcrete())
 	envTest = &envtest.Environment{
 		BinaryAssetsDirectory: filepath.Join(envTestDir, "k8s", versionDir),
+		CRDs: []*apiextensionsv1spec.CustomResourceDefinition{
+			CRD("project.openshift.io", "v1", "projects", "Project", "project", false),
+			CRD("route.openshift.io", "v1", "routes", "Route", "route", true),
+		},
 	}
+	// Configure API server for faster CRD establishment and test performance
+	envTest.ControlPlane.GetAPIServer().Configure().
+		// Increase concurrent request limits for faster parallel operations
+		Set("max-requests-inflight", "1000").
+		Set("max-mutating-requests-inflight", "500").
+		// Speed up namespace cleanup with more workers
+		Set("delete-collection-workers", "10") //.
+	// Enable verbose logging for debugging
+	//Set("v", "9")
+
 	adminSystemMasterBaseConfig, _ := envTest.Start()
 	au := test.Must(envTest.AddUser(envTestUser, adminSystemMasterBaseConfig))
 	envTestRestConfig = au.Config()
@@ -195,98 +202,3 @@ func (s *BaseMcpSuite) InitMcpClient(options ...transport.StreamableHTTPCOption)
 	s.Require().NoError(err, "Expected no error creating MCP server")
 	s.McpClient = test.NewMcpClient(s.T(), s.mcpServer.ServeHTTP(), options...)
 }
-
-// EnvTestInOpenShift sets up the kubernetes environment to seem to be running OpenShift
-func EnvTestInOpenShift(ctx context.Context) error {
-	crdTemplate := `
-          {
-            "apiVersion": "apiextensions.k8s.io/v1",
-            "kind": "CustomResourceDefinition",
-            "metadata": {"name": "%s"},
-            "spec": {
-              "group": "%s",
-              "versions": [{
-                "name": "v1","served": true,"storage": true,
-                "schema": {"openAPIV3Schema": {"type": "object","x-kubernetes-preserve-unknown-fields": true}}
-              }],
-              "scope": "%s",
-              "names": {"plural": "%s","singular": "%s","kind": "%s"}
-            }
-          }`
-	tasks, _ := errgroup.WithContext(ctx)
-	tasks.Go(func() error {
-		return EnvTestCrdApply(ctx, fmt.Sprintf(crdTemplate, "projects.project.openshift.io", "project.openshift.io",
-			"Cluster", "projects", "project", "Project"))
-	})
-	tasks.Go(func() error {
-		return EnvTestCrdApply(ctx, fmt.Sprintf(crdTemplate, "routes.route.openshift.io", "route.openshift.io",
-			"Namespaced", "routes", "route", "Route"))
-	})
-	return tasks.Wait()
-}
-
-// EnvTestInOpenShiftClear clears the kubernetes environment so it no longer seems to be running OpenShift
-func EnvTestInOpenShiftClear(ctx context.Context) error {
-	tasks, _ := errgroup.WithContext(ctx)
-	tasks.Go(func() error { return EnvTestCrdDelete(ctx, "projects.project.openshift.io") })
-	tasks.Go(func() error { return EnvTestCrdDelete(ctx, "routes.route.openshift.io") })
-	return tasks.Wait()
-}
-
-// EnvTestCrdWaitUntilReady waits for a CRD to be established
-func EnvTestCrdWaitUntilReady(ctx context.Context, name string) error {
-	apiExtensionClient := apiextensionsv1.NewForConfigOrDie(envTestRestConfig)
-	watcher, err := apiExtensionClient.CustomResourceDefinitions().Watch(ctx, metav1.ListOptions{
-		FieldSelector: "metadata.name=" + name,
-	})
-	if err != nil {
-		return fmt.Errorf("unable to watch CRDs: %w", err)
-	}
-	_, err = toolswatch.UntilWithoutRetry(ctx, watcher, func(event watch.Event) (bool, error) {
-		for _, c := range event.Object.(*apiextensionsv1spec.CustomResourceDefinition).Status.Conditions {
-			if c.Type == apiextensionsv1spec.Established && c.Status == apiextensionsv1spec.ConditionTrue {
-				return true, nil
-			}
-		}
-		return false, nil
-	})
-	if err != nil {
-		return fmt.Errorf("failed to wait for CRD: %w", err)
-	}
-	return nil
-}
-
-// EnvTestCrdApply creates a CRD from the provided resource string and waits for it to be established
-func EnvTestCrdApply(ctx context.Context, resource string) error {
-	apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig)
-	var crd = &apiextensionsv1spec.CustomResourceDefinition{}
-	err := json.Unmarshal([]byte(resource), crd)
-	if err != nil {
-		return fmt.Errorf("failed to create CRD %v", err)
-	}
-	_, err = apiExtensionsV1Client.CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{})
-	if err != nil {
-		return fmt.Errorf("failed to create CRD %v", err)
-	}
-	return EnvTestCrdWaitUntilReady(ctx, crd.Name)
-}
-
-// crdDelete deletes a CRD by name and waits for it to be removed
-func EnvTestCrdDelete(ctx context.Context, name string) error {
-	apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig)
-	err := apiExtensionsV1Client.CustomResourceDefinitions().Delete(ctx, name, metav1.DeleteOptions{
-		GracePeriodSeconds: ptr.To(int64(0)),
-	})
-	iteration := 0
-	for iteration < 100 {
-		if _, derr := apiExtensionsV1Client.CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{}); derr != nil {
-			break
-		}
-		time.Sleep(5 * time.Millisecond)
-		iteration++
-	}
-	if err != nil {
-		return errors.Wrap(err, "failed to delete CRD")
-	}
-	return nil
-}

pkg/mcp/resources_test.go

@@ -401,7 +401,7 @@ func (s *ResourcesSuite) TestResourcesCreateOrUpdate() {
 			_, err = apiExtensionsV1Client.CustomResourceDefinitions().Get(s.T().Context(), "customs.example.com", metav1.GetOptions{})
 			s.Nilf(err, "custom resource definition not found")
 		})
-		s.Require().NoError(EnvTestCrdWaitUntilReady(s.T().Context(), "customs.example.com"))
+		s.Require().NoError(EnvTestWaitForAPIResourceCondition(s.T().Context(), "example.com", "v1", "customs", true))
 	})
 
 	s.Run("resources_create_or_update creates custom resource", func() {