K8SPSMDB-1419: Add replset remapping support (#2147)

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: egegunes

config/crd/bases/psmdb.percona.com_perconaservermongodbrestores.yaml

@@ -229,8 +229,10 @@ spec:
                     ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]$'))
                 - message: Date should not be used when 'latest' type is used
                   rule: self.type != 'latest' || !has(self.date)
-              replset:
-                type: string
+              replsetRemapping:
+                additionalProperties:
+                  type: string
+                type: object
               selective:
                 properties:
                   namespaces:

deploy/backup/restore.yaml

@@ -9,6 +9,9 @@ spec:
 #    withUsersAndRoles: true
 #    namespaces:
 #    - "db.collection"
+#  replsetRemapping:
+#    rs0: shard0
+#    rs1: shard1
 #  pitr:
 #    type: date
 #    date: YYYY-MM-DD HH:MM:SS

deploy/bundle.yaml

@@ -495,8 +495,10 @@ spec:
                     ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]$'))
                 - message: Date should not be used when 'latest' type is used
                   rule: self.type != 'latest' || !has(self.date)
-              replset:
-                type: string
+              replsetRemapping:
+                additionalProperties:
+                  type: string
+                type: object
               selective:
                 properties:
                   namespaces:

deploy/crd.yaml

@@ -495,8 +495,10 @@ spec:
                     ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]$'))
                 - message: Date should not be used when 'latest' type is used
                   rule: self.type != 'latest' || !has(self.date)
-              replset:
-                type: string
+              replsetRemapping:
+                additionalProperties:
+                  type: string
+                type: object
               selective:
                 properties:
                   namespaces:

deploy/cw-bundle.yaml

@@ -495,8 +495,10 @@ spec:
                     ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]$'))
                 - message: Date should not be used when 'latest' type is used
                   rule: self.type != 'latest' || !has(self.date)
-              replset:
-                type: string
+              replsetRemapping:
+                additionalProperties:
+                  type: string
+                type: object
               selective:
                 properties:
                   namespaces:

e2e-tests/functions

@@ -308,6 +308,8 @@ wait_for_pbm_operations() {
 run_restore() {
 	local backup_name=$1
 
+	log "running restore restore-$backup_name"
+
 	cat $test_dir/conf/restore.yml \
 		| $sed -e "s/name:/name: restore-$backup_name/" \
 		| $sed -e "s/backupName:/backupName: $backup_name/" \
@@ -939,9 +941,13 @@ run_mongo() {
 	local uri="$2"
 	local driver=${3:-mongodb+srv}
 	local suffix=${4:-.svc.cluster.local}
-	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')
 	local mongo_flag="$5"
-	[[ $uri == *cfg* ]] && replica_set='cfg' || replica_set='rs0'
+	local replica_set=${6:-"rs0"}
+	if [[ $uri == *cfg* ]]; then
+		replica_set="cfg"
+	fi
+
+	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')
 	kubectl_bin exec ${client_container} -- \
 		bash -c "printf '$command\n' | mongo $driver://$uri$suffix/admin?ssl=false\&replicaSet=$replica_set $mongo_flag"
 }
@@ -1080,6 +1086,7 @@ compare_mongo_cmd() {
 	local collection="${6:-test}"
 	local sort="$7"
 	local tls="${8:-false}"
+	local replicaset="$9"
 
 	local full_command="db.${collection}.${command}()"
 	if [[ -n ${sort} ]]; then
@@ -1094,7 +1101,7 @@ compare_mongo_cmd() {
 		mongo_command=run_mongo
 	fi
 
-	$mongo_command "use ${database}\n ${full_command}" "$uri" "mongodb" "$suffix" \
+	$mongo_command "use ${database}\n ${full_command}" "$uri" "mongodb" "$suffix" "" ${replicaset} \
 		| egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' \
 		| $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' \
 			>$tmp_dir/${command}${postfix}
@@ -1122,6 +1129,8 @@ compare_mongos_cmd() {
 		mongos_command=run_mongos
 	fi
 
+	log "running db.${collection}.command() in ${database}"
+
 	$mongos_command "use ${database}\n db.${collection}.${command}()" "$uri" "mongodb" "$suffix" "" "$port" \
 		| egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' \
 		| $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' \
@@ -1560,7 +1569,7 @@ run_backup() {
 	local backup_name=${2:-"backup-${storage}"}
 	local type=${3:-"logical"}
 
-	desc "run backup $backup_name"
+	log "running backup $backup_name"
 
 	yq eval '.metadata.name = "'${backup_name}'"
 			| .spec.storageName = "'${storage}'"

e2e-tests/replset-remapping-sharded/compare/find.json

@@ -0,0 +1,3 @@
+switched to db myApp
+{ "_id" : , "x" : 100500 }
+bye

e2e-tests/replset-remapping-sharded/conf/backup-minio.yml

@@ -0,0 +1,9 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDBBackup
+metadata:
+  name:
+  finalizers:
+    - percona.com/delete-backup
+spec:
+  clusterName: some-name-source
+  storageName: minio

e2e-tests/replset-remapping-sharded/conf/restore.yml

@@ -0,0 +1,11 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDBRestore
+metadata:
+  name:
+spec:
+  clusterName: some-name-target
+  backupName:
+  replsetRemapping:
+    rs0: replset0
+    rs1: replset1
+    cfg: csReplSet

e2e-tests/replset-remapping-sharded/conf/some-name-source.yml

@@ -0,0 +1,84 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  finalizers:
+  - percona.com/delete-psmdb-pvc
+  name: some-name-source
+spec:
+  image: perconalab/percona-server-mongodb-operator:main-mongod8.0
+  imagePullPolicy: Always
+  updateStrategy: SmartUpdate
+  secrets:
+    users: some-users
+  backup:
+    enabled: true
+    image: perconalab/percona-server-mongodb-operator:main-backup
+    storages:
+      minio:
+        type: s3
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing
+          endpointUrl: http://minio-service:9000/
+          insecureSkipTLSVerify: false
+  sharding:
+    enabled: true
+    configsvrReplSet:
+      size: 3
+      resources:
+        limits:
+          cpu: "300m"
+          memory: "0.5G"
+        requests:
+          cpu: "300m"
+          memory: "0.5G"
+      volumeSpec:
+        persistentVolumeClaim:
+          resources:
+            requests:
+              storage: 3Gi
+    mongos:
+      size: 3
+      resources:
+        limits:
+          cpu: "300m"
+          memory: "0.5G"
+        requests:
+          cpu: "300m"
+          memory: "0.5G"
+  replsets:
+  - name: rs0
+    size: 3
+    expose:
+      enabled: false
+      exposeType: ClusterIP
+    resources:
+      limits:
+        cpu: 300m
+        memory: 0.5G
+      requests:
+        cpu: 300m
+        memory: 0.5G
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 3Gi
+  - name: rs1
+    size: 3
+    expose:
+      enabled: false
+      exposeType: ClusterIP
+    resources:
+      limits:
+        cpu: 300m
+        memory: 0.5G
+      requests:
+        cpu: 300m
+        memory: 0.5G
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 3Gi