From 1bf9e34295aab832315a565fd35351e4e196a6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 16 Dec 2025 15:23:04 +0300 Subject: [PATCH 1/4] K8SPSMDB-1451: Wait until primary elected after rs initialization --- pkg/controller/perconaservermongodb/mgo.go | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/pkg/controller/perconaservermongodb/mgo.go b/pkg/controller/perconaservermongodb/mgo.go index 9905c4d4dd..e0180d747d 100644 --- a/pkg/controller/perconaservermongodb/mgo.go +++ b/pkg/controller/perconaservermongodb/mgo.go @@ -17,6 +17,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" logf "sigs.k8s.io/controller-runtime/pkg/log" api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" @@ -719,8 +721,35 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c return nil, nil, fmt.Errorf("exec rs.initiate: %v / %s / %s", err, outb.String(), errb.String()) } + backoff := wait.Backoff{ + Steps: 5, + Duration: 50 * time.Millisecond, + Factor: 5.0, + Jitter: 0.1, + } + err = retry.OnError(backoff, func(err error) bool { return true }, func() error { + var stderr, stdout bytes.Buffer + + hello := []string{"sh", "-c", + mongoCmd + " --eval 'db.runCommand({ hello: 1 }).isWritablePrimary'"} + err = r.clientcmd.Exec(ctx, &pod, "mongod", hello, nil, &stdout, &stderr, false) + if err != nil { + return errors.Wrapf(err, "run hello stdout: %s, stderr: %s", stdout.String(), stderr.String()) + } + + out := strings.Trim(stdout.String(), "\n") + if out != "true" { + return errors.New("is not the writable primary") + } + + log.Info(pod.Name+" is the writable primary", "replset", replsetName) + + return nil + }) + if err != nil { + return nil, nil, errors.Wrap(err, "wait for replset initialization") + } log.Info("replset initialized", "replset", replsetName, "pod", pod.Name) - time.Sleep(time.Second * 5) log.Info("creating user admin", "replset", replsetName, "pod", pod.Name, "user", api.RoleUserAdmin) userAdmin, err := getInternalCredentials(ctx, r.client, cr, api.RoleUserAdmin) From c6842046be02cbb0376dd6e9ac212be6ee0d16da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 18 Dec 2025 15:19:01 +0300 Subject: [PATCH 2/4] address review comments --- pkg/controller/perconaservermongodb/mgo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/perconaservermongodb/mgo.go b/pkg/controller/perconaservermongodb/mgo.go index e0180d747d..96828c2b37 100644 --- a/pkg/controller/perconaservermongodb/mgo.go +++ b/pkg/controller/perconaservermongodb/mgo.go @@ -731,7 +731,7 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c var stderr, stdout bytes.Buffer hello := []string{"sh", "-c", - mongoCmd + " --eval 'db.runCommand({ hello: 1 }).isWritablePrimary'"} + mongoCmd + " --quiet --eval 'db.runCommand({ hello: 1 }).isWritablePrimary'"} err = r.clientcmd.Exec(ctx, &pod, "mongod", hello, nil, &stdout, &stderr, false) if err != nil { return errors.Wrapf(err, "run hello stdout: %s, stderr: %s", stdout.String(), stderr.String()) @@ -739,7 +739,7 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c out := strings.Trim(stdout.String(), "\n") if out != "true" { - return errors.New("is not the writable primary") + return errors.New(pod.Name + " is not the writable primary") } log.Info(pod.Name+" is the writable primary", "replset", replsetName) From 8b57e5406092621442e398769af03e227878528a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 18 Dec 2025 15:30:17 +0300 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/controller/perconaservermongodb/mgo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/perconaservermongodb/mgo.go b/pkg/controller/perconaservermongodb/mgo.go index 96828c2b37..a1522c692c 100644 --- a/pkg/controller/perconaservermongodb/mgo.go +++ b/pkg/controller/perconaservermongodb/mgo.go @@ -737,9 +737,9 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c return errors.Wrapf(err, "run hello stdout: %s, stderr: %s", stdout.String(), stderr.String()) } - out := strings.Trim(stdout.String(), "\n") + out := strings.TrimSpace(stdout.String()) if out != "true" { - return errors.New(pod.Name + " is not the writable primary") + return errors.Errorf("%s is not the writable primary", pod.Name) } log.Info(pod.Name+" is the writable primary", "replset", replsetName) From 65db8e006fa31d7a31c50d79869581e9e23f355d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 18 Dec 2025 15:45:02 +0300 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/controller/perconaservermongodb/mgo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/perconaservermongodb/mgo.go b/pkg/controller/perconaservermongodb/mgo.go index a1522c692c..aaa11b9009 100644 --- a/pkg/controller/perconaservermongodb/mgo.go +++ b/pkg/controller/perconaservermongodb/mgo.go @@ -731,8 +731,8 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c var stderr, stdout bytes.Buffer hello := []string{"sh", "-c", - mongoCmd + " --quiet --eval 'db.runCommand({ hello: 1 }).isWritablePrimary'"} - err = r.clientcmd.Exec(ctx, &pod, "mongod", hello, nil, &stdout, &stderr, false) + mongoCmd + " --quiet --eval 'db.hello().isWritablePrimary'"} + err := r.clientcmd.Exec(ctx, &pod, "mongod", hello, nil, &stdout, &stderr, false) if err != nil { return errors.Wrapf(err, "run hello stdout: %s, stderr: %s", stdout.String(), stderr.String()) }