SERVER-42299 Upgrade/downgrade for config.chunks and config.tags

Author: jsmulrow

jstests/multiVersion/config_chunks_tags_set_fcv.js

@@ -0,0 +1,142 @@
+/**
+ * Tests that config.chunks and config.tags documents are correctly modified on FCV
+ * upgrade/downgrade.
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/parallelTester.js");  // for Thread.
+load("jstests/multiVersion/libs/config_chunks_tags_shared.js");
+load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
+// Assumes ns has the following chunk layout: [-inf, -50), [-50, 0) on shard0 and [0, inf) on
+// shard 1.
+function verifyChunkOperationsFailDuringSetFCV(st, ns) {
+    assert.commandFailedWithCode(st.s.adminCommand({split: ns, middle: {_id: 50}}),
+                                 ErrorCodes.ConflictingOperationInProgress);
+    verifyChunkDistribution(st, ns, [2, 1]);
+
+    assert.commandFailedWithCode(
+        st.s.adminCommand({moveChunk: ns, find: {_id: 0}, to: st.shard0.shardName}),
+        ErrorCodes.ConflictingOperationInProgress);
+    verifyChunkDistribution(st, ns, [2, 1]);
+
+    assert.commandFailedWithCode(
+        st.s.adminCommand({mergeChunks: ns, bounds: [{_id: MinKey}, {_id: 0}]}),
+        ErrorCodes.ConflictingOperationInProgress);
+    verifyChunkDistribution(st, ns, [2, 1]);
+}
+
+// Assumes shard0 is in zone0 which contains [-inf, 0) and is not in zone1.
+function verifyZoneOperationsSucceedDuringSetFCV(st, ns) {
+    assert.commandWorked(
+        st.s.adminCommand({updateZoneKeyRange: ns, min: {_id: MinKey}, max: {_id: 0}, zone: null}));
+    assert.commandWorked(st.s.adminCommand(
+        {updateZoneKeyRange: ns, min: {_id: MinKey}, max: {_id: 0}, zone: "zone0"}));
+
+    assert.commandWorked(st.s.adminCommand({addShardToZone: st.shard0.shardName, zone: "zone1"}));
+    assert.commandWorked(
+        st.s.adminCommand({removeShardFromZone: st.shard0.shardName, zone: "zone1"}));
+}
+
+const dbName = "test";
+const chunkNs = dbName + ".chunk_coll";
+const zoneNs = dbName + ".zone_coll";
+
+const st = new ShardingTest({shards: 2});
+const configPrimary = st.configRS.getPrimary();
+
+assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+st.ensurePrimaryShard(dbName, st.shard0.shardName);
+
+setUpCollectionForChunksTesting(st, chunkNs);
+setUpCollectionForZoneTesting(st, zoneNs);
+
+//
+// Verify chunk and tag documents are updated by setFeatureCompatibilityVersion.
+//
+
+checkFCV(configPrimary.getDB("admin"), latestFCV);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: true});
+
+jsTestLog("Downgrading FCV to last stable");
+assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
+checkFCV(configPrimary.getDB("admin"), lastStableFCV);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Upgrading FCV to latest");
+assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+checkFCV(configPrimary.getDB("admin"), latestFCV);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: true});
+
+//
+// Verify operations during setFeatureCompabitilityVersion use the correct format and that setFCV
+// blocks behind in-progress shard collections on shard servers.
+//
+
+function runInProgressSetFCVTest(st, {initialFCV, desiredFCV}) {
+    const pauseInSetFCVFailPointName = desiredFCV === lastStableFCV
+        ? "pauseBeforeDowngradingConfigMetadata"
+        : "pauseBeforeUpgradingConfigMetadata";
+
+    clearRawMongoProgramOutput();
+    checkFCV(configPrimary.getDB("admin"), initialFCV);
+
+    // Pause setFCV to test the in-progress states.
+    assert.commandWorked(configPrimary.adminCommand(
+        {configureFailPoint: pauseInSetFCVFailPointName, mode: "alwaysOn"}));
+
+    // Start and pause a shard collection, and verify that the setFCV blocks behind it.
+    const shardCollDuringSetFCV = new Thread((host, ns) => {
+        const mongosConn = new Mongo(host);
+        return mongosConn.adminCommand({shardCollection: ns, key: {_id: 1}});
+    }, st.s.host, dbName + ".setFCVTo" + desiredFCV);
+    assert.commandWorked(st.rs0.getPrimary().adminCommand(
+        {configureFailPoint: "pauseShardCollectionBeforeReturning", mode: "alwaysOn"}));
+    shardCollDuringSetFCV.start();
+    waitForFailpoint("Hit pauseShardCollectionBeforeReturning", 1 /* numTimes */);
+
+    // Assert setFCV can't hit the failpoint until the shard collection completes.
+    const changeFCV = new Thread((host, fcv) => {
+        const mongosConn = new Mongo(host);
+        return mongosConn.adminCommand({setFeatureCompatibilityVersion: fcv});
+    }, st.s.host, desiredFCV);
+    changeFCV.start();
+    assert.throws(() => {
+        waitForFailpoint("Hit " + pauseInSetFCVFailPointName, 1 /* numTimes */, 3000 /* timeout */);
+    });
+
+    // Unpause the shard collection and wait for setFCV to reach the failpoint.
+    assert.commandWorked(st.rs0.getPrimary().adminCommand(
+        {configureFailPoint: "pauseShardCollectionBeforeReturning", mode: "off"}));
+    shardCollDuringSetFCV.join();
+    waitForFailpoint("Hit " + pauseInSetFCVFailPointName, 1 /* numTimes */);
+
+    // Verify behavior while setFCV is in progress.
+    verifyChunkOperationsFailDuringSetFCV(st, chunkNs);
+    verifyZoneOperationsSucceedDuringSetFCV(st, zoneNs);
+    testCRUDOperations(st, chunkNs);
+    testCRUDOperations(st, zoneNs);
+
+    // A collection can still be sharded during setFCV and should write chunks correctly.
+    verifyInitialChunks(
+        st, dbName + ".newCollDuringFCV" + desiredFCV, {expectNewFormat: desiredFCV === latestFCV});
+
+    // Unset the setFCV failpoint and allow setFCV to finish.
+    assert.commandWorked(
+        configPrimary.adminCommand({configureFailPoint: pauseInSetFCVFailPointName, mode: "off"}));
+    changeFCV.join();
+    assert.commandWorked(changeFCV.returnData());
+    checkFCV(configPrimary.getDB("admin"), desiredFCV);
+
+    verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: desiredFCV === latestFCV});
+}
+
+runInProgressSetFCVTest(st, {initialFCV: latestFCV, desiredFCV: lastStableFCV});
+runInProgressSetFCVTest(st, {initialFCV: lastStableFCV, desiredFCV: latestFCV});
+
+st.stop();
+}());

jstests/multiVersion/config_chunks_tags_upgrade_downgrade_cluster.js

@@ -0,0 +1,124 @@
+/**
+ * Tests upgrading a cluster from last stable to the latest version and downgrading it back to last
+ * stable, verifying the behavior of chunk and zone operations throughout the process.
+ */
+
+// Checking UUID consistency uses cached connections, which are not valid across restarts or
+// stepdowns.
+TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+
+(function() {
+"use strict";
+
+load("jstests/multiVersion/libs/config_chunks_tags_shared.js");
+load("jstests/multiVersion/libs/multi_cluster.js");
+load("jstests/multiVersion/libs/multi_rs.js");
+
+// Runs commands on the config server that will use its RSM to target both shard primaries until
+// they succeed.
+function waitForConfigServerShardRSMRetarget(st) {
+    assert.soonNoExcept(() => {
+        assert.commandWorked(st.s.getDB("unrelated_db").unrelated_coll.insert({x: 1}));
+        st.ensurePrimaryShard("unrelated_db", st.shard0.shardName);
+        st.ensurePrimaryShard("unrelated_db", st.shard1.shardName);
+        st.ensurePrimaryShard("unrelated_db", st.shard0.shardName);
+        assert.commandWorked(st.s.getDB("unrelated_db").dropDatabase());
+        return true;
+    });
+}
+
+const dbName = "test";
+const chunkNs = dbName + ".chunk_coll";
+const zoneNs = dbName + ".zone_coll";
+
+// Start a cluster with two shards at the last stable version and a sharding enabled db.
+const st = new ShardingTest({
+    shards: 2,
+    other: {
+        mongosOptions: {binVersion: "last-stable"},
+        configOptions: {binVersion: "last-stable"},
+        rsOptions: {binVersion: "last-stable"},
+    },
+    rs: {nodes: 3}  // Use 3 node replica sets to allow binary changes with no downtime.
+});
+checkFCV(st.configRS.getPrimary().getDB("admin"), lastStableFCV);
+assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+st.ensurePrimaryShard(dbName, st.shard0.shardName);
+
+// Set up sharded collections for targeted chunk and zone operation testing.
+setUpCollectionForChunksTesting(st, chunkNs);
+setUpCollectionForZoneTesting(st, zoneNs);
+
+// Set up another sharded collection on a different database to verify chunks and zones are updated
+// for every sharded collection.
+setUpExtraShardedCollections(st, "extra_db" /* dbName */);
+
+//
+// Upgrade back to the latest version.
+//
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Upgrading config servers.");
+st.upgradeCluster("latest", {upgradeConfigs: true, upgradeMongos: false, upgradeShards: false});
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Upgrading shard servers.");
+st.upgradeCluster("latest", {upgradeConfigs: false, upgradeMongos: false, upgradeShards: true});
+
+// Manually moving a chunk will use the config server's replica set monitor to target the primary of
+// the source shard. After upgrading the shard servers above, this RSM may be stale, so run
+// operations through the config server that will use the same RSM so it picks up the new primary.
+waitForConfigServerShardRSMRetarget(st);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Upgrading mongos servers.");
+st.upgradeCluster("latest", {upgradeConfigs: false, upgradeMongos: true, upgradeShards: false});
+checkFCV(st.configRS.getPrimary().getDB("admin"), lastStableFCV);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Upgrade feature compatibility version to latest");
+assert.commandWorked(st.s.getDB("admin").runCommand({setFeatureCompatibilityVersion: latestFCV}));
+checkFCV(st.configRS.getPrimary().getDB("admin"), latestFCV);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: true});
+
+//
+// Downgrade back to the last stable version.
+//
+
+jsTestLog("Downgrade feature compatibility version to last stable");
+assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
+checkFCV(st.configRS.getPrimary().getDB("admin"), lastStableFCV);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Downgrading mongos servers.");
+st.upgradeCluster("last-stable",
+                  {upgradeConfigs: false, upgradeMongos: true, upgradeShards: false});
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Downgrading shard servers.");
+st.upgradeCluster("last-stable",
+                  {upgradeConfigs: false, upgradeMongos: false, upgradeShards: true});
+
+// Manually moving a chunk will use the config server's replica set monitor to target the primary of
+// the source shard. After upgrading the shard servers above, this RSM may be stale, so run
+// operations through the config server that will use the same RSM so it picks up the new primary.
+waitForConfigServerShardRSMRetarget(st);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+jsTestLog("Downgrading config servers.");
+st.upgradeCluster("last-stable",
+                  {upgradeConfigs: true, upgradeMongos: false, upgradeShards: false});
+checkFCV(st.configRS.getPrimary().getDB("admin"), lastStableFCV);
+
+verifyChunksAndTags(st, dbName, chunkNs, zoneNs, {expectNewFormat: false});
+
+st.stop();
+})();