reconnect cluster when kubeconfig secret changed

Author: qliang

controllers/clustercache/cluster_accessor.go

@@ -154,6 +154,11 @@ type clusterAccessorHealthProbeConfig struct {
 // and health checking information (e.g. lastProbeSuccessTimestamp, consecutiveFailures).
 // lockedStateLock must be *always* held (via lock or rLock) before accessing this field.
 type clusterAccessorLockedState struct {
+	// kubeconfigResourceVersion is the resource version of the kubeconfig secret.
+	// This is used to detect if the kubeconfig secret has changed and we need to re-create the connection.
+	// It is set when the connection is created.
+	kubeconfigResourceVersion string
+
 	// lastConnectionCreationErrorTimestamp is the timestamp when connection creation failed the last time.
 	lastConnectionCreationErrorTimestamp time.Time
 
@@ -273,6 +278,12 @@ func (ca *clusterAccessor) Connect(ctx context.Context) (retErr error) {
 
 	log.Info("Connected")
 
+	kubeconfigSecret, err := ca.getKubeConfigSecret(ctx)
+	if err != nil {
+		return err
+	}
+	ca.lockedState.kubeconfigResourceVersion = kubeconfigSecret.ResourceVersion
+
 	// Only generate the clientCertificatePrivateKey once as there is no need to regenerate it after disconnect/connect.
 	// Note: This has to be done before setting connection, because otherwise this code wouldn't be re-entrant if the
 	// private key generation fails because we check Connected above.
@@ -414,6 +425,18 @@ func (ca *clusterAccessor) GetRESTConfig(ctx context.Context) (*rest.Config, err
 	return ca.lockedState.connection.restConfig, nil
 }
 
+func (ca *clusterAccessor) KubeConfigUpdated(ctx context.Context) (bool, error) {
+	ca.rLock(ctx)
+	defer ca.rUnlock(ctx)
+
+	kubeconfigSecret, err := ca.getKubeConfigSecret(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	return kubeconfigSecret.ResourceVersion != ca.lockedState.kubeconfigResourceVersion, nil
+}
+
 func (ca *clusterAccessor) GetRESTConfigFromSecret(ctx context.Context) (*rest.Config, error) {
 	ca.rLock(ctx)
 	defer ca.rUnlock(ctx)

controllers/clustercache/cluster_accessor_client.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -39,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
 	kcfg "sigs.k8s.io/cluster-api/util/kubeconfig"
+	"sigs.k8s.io/cluster-api/util/secret"
 )
 
 type createConnectionResult struct {
@@ -48,6 +50,14 @@ type createConnectionResult struct {
 	Cache        *stoppableCache
 }
 
+func (ca *clusterAccessor) getKubeConfigSecret(ctx context.Context) (*v1.Secret, error) {
+	kubeconfigSecret, err := secret.Get(ctx, ca.config.SecretClient, ca.cluster, secret.Kubeconfig)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting kubeconfig secret")
+	}
+	return kubeconfigSecret, nil
+}
+
 func (ca *clusterAccessor) createConnection(ctx context.Context) (*createConnectionResult, error) {
 	log := ctrl.LoggerFrom(ctx)
 	log.V(6).Info("Creating connection")

controllers/clustercache/cluster_cache.go

@@ -460,8 +460,20 @@ func (cc *clusterCache) Reconcile(ctx context.Context, req reconcile.Request) (r
 
 	requeueAfterDurations := []time.Duration{}
 
+	kubeconfigUpdated, err := accessor.KubeConfigUpdated(ctx)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "error checking if kubeconfig was updated for cluster %s/%s", clusterKey.Namespace, clusterKey.Name)
+	}
+
 	// Try to connect, if not connected.
 	connected := accessor.Connected(ctx)
+	if connected && kubeconfigUpdated {
+		log.Info("Kubeconfig was updated, disconnecting to re-connect with the new kubeconfig")
+		accessor.Disconnect(ctx)
+		didDisconnect = true
+		connected = false
+	}
+
 	if !connected {
 		lastConnectionCreationErrorTimestamp := accessor.GetLastConnectionCreationErrorTimestamp(ctx)
 

controlplane/kubeadm/internal/cluster.go

@@ -99,7 +99,7 @@ func (m *Management) GetMachinePoolsForCluster(ctx context.Context, cluster *clu
 func (m *Management) GetWorkloadCluster(ctx context.Context, clusterKey client.ObjectKey) (WorkloadCluster, error) {
 	// TODO(chuckha): Inject this dependency.
 	// TODO(chuckha): memoize this function. The workload client only exists as long as a reconciliation loop.
-	restConfig, err := m.ClusterCache.GetRESTConfigFromSecret(ctx, clusterKey)
+	restConfig, err := m.ClusterCache.GetRESTConfig(ctx, clusterKey)
 	if err != nil {
 		return nil, &RemoteClusterConnectionError{Name: clusterKey.String(), Err: err}
 	}