diff --git a/jstests/replsets/auto_promote_hidden_test.js b/jstests/replsets/auto_promote_hidden_test.js new file mode 100644 index 0000000000000..476ba42063196 --- /dev/null +++ b/jstests/replsets/auto_promote_hidden_test.js @@ -0,0 +1,561 @@ +/** + * Tests autoPromoteHidden functionality with read preference scenarios. + * + * Case 1: 3-member replica set with 1 hidden node + * - Test read preference secondary only fails when non-hidden secondaries are down + * - Test read preference primary succeeds + * - Test with autoPromoteHidden enabled, secondary only reads succeed + * + * Case 2: Sharded cluster with autoPromoteHidden + * - Similar to Case 1 but in sharded environment + * + * Case 3: 5-member replica set with 2 hidden nodes + * - Stop 2 non-hidden secondaries and test functionality + * + * @tags: [requires_fcv_81] + */ + +import {reconfig} from "jstests/replsets/rslib.js"; + +const dbName = "testDB"; +const collName = "testColl"; + +// Helper function to test read with specific read preference +function testRead(conn, dbName, collName, readPref, shouldSucceed, description) { + try { + const result = conn.getDB(dbName).runCommand({ + find: collName, + limit: 1, + $readPreference: readPref + }); + + if (shouldSucceed) { + assert.commandWorked(result, description + " - should succeed but failed"); + jsTestLog("[PASS] " + description + " - succeeded as expected"); + return true; + } else { + assert.commandFailed(result, description + " - should fail but succeeded"); + jsTestLog("[FAIL] " + description + " - unexpectedly succeeded"); + return false; + } + } catch (e) { + if (!shouldSucceed) { + jsTestLog("[PASS] " + description + " - failed as expected: " + e.message); + return true; + } else { + jsTestLog("[FAIL] " + description + " - unexpectedly failed: " + e.message); + throw e; + } + } +} + +// ============================================================================= +// CASE 1: 3-member replica set with 1 hidden node +// ============================================================================= +(function testCase1_ThreeNodeReplSet() { + jsTestLog("===================================================================="); + jsTestLog("CASE 1: Testing 3-member replica set with 1 hidden node"); + jsTestLog("===================================================================="); + + const rst = new ReplSetTest({ + name: "auto_promote_hidden_case1", + nodes: [ + {}, // Primary + {}, // Secondary 1 + {rsConfig: {priority: 0, hidden: true}} // Hidden node + ], + settings: {heartbeatIntervalMillis: 500, electionTimeoutMillis: 2000} + }); + + rst.startSet(); + rst.initiate(); + + const primary = rst.getPrimary(); + const secondaries = rst.getSecondaries(); + const regularSecondary = secondaries[0]; + const hiddenNode = secondaries[1]; + + jsTestLog("Setting up test data..."); + assert.commandWorked(primary.getDB(dbName).getCollection(collName).insert( + [{_id: 1, data: "test"}] + )); + rst.awaitReplication(); + + jsTestLog("Step 1.1: Verify initial read preferences work correctly"); + // Read from primary should work + testRead(primary, dbName, collName, {mode: "primary"}, true, + "Read with primary preference"); + + // Read from secondary should work (we have 1 non-hidden secondary available) + testRead(primary, dbName, collName, {mode: "secondary"}, true, + "Read with secondary preference (before stopping secondary)"); + + jsTestLog("Step 1.2: Stop the non-hidden secondary"); + rst.stop(regularSecondary); + + // Wait for primary to detect the secondary is down + assert.soon(() => { + const status = assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})); + for (let member of status.members) { + if (member.name === regularSecondary.host) { + return member.health === 0; + } + } + return false; + }, "Primary should detect secondary is down", 30000); + + sleep(3000); + + jsTestLog("Step 1.3: Test read preferences with autoPromoteHidden=false (default)"); + // Read from primary should still work + testRead(primary, dbName, collName, {mode: "primary"}, true, + "Read with primary preference (secondary down, autoPromote=false)"); + + // Read from secondary should fail - no non-hidden secondary available + // Hidden node is not visible for read preference routing + testRead(primary, dbName, collName, {mode: "secondary"}, false, + "Read with secondary preference (secondary down, autoPromote=false)"); + + jsTestLog("Step 1.4: Enable autoPromoteHidden"); + let config = rst.getReplSetConfigFromNode(); + config.version++; + config.settings = config.settings || {}; + config.settings.autoPromoteHidden = true; + + assert.commandWorked(primary.adminCommand({replSetReconfig: config})); + jsTestLog("Reconfig command worked - autoPromoteHidden enabled"); + + // Wait for config to propagate and topology to update + sleep(5000); + + // Verify hidden node is now in hosts list + const helloResp = assert.commandWorked(primary.adminCommand({hello: 1})); + jsTestLog("Hello response after enabling autoPromoteHidden: " + tojson(helloResp)); + + let hiddenPromoted = helloResp.hosts && helloResp.hosts.includes(hiddenNode.host); + assert.eq(true, hiddenPromoted, + "Hidden node should be in hosts list after enabling autoPromoteHidden"); + + jsTestLog("Step 1.5: Test read preferences with autoPromoteHidden=true"); + // Read from primary should still work + testRead(primary, dbName, collName, {mode: "primary"}, true, + "Read with primary preference (autoPromote=true)"); + + // Read from secondary should now succeed because hidden node is promoted + testRead(primary, dbName, collName, {mode: "secondary"}, true, + "Read with secondary preference (secondary down, autoPromote=true)"); + + jsTestLog("Step 1.6: Restart the secondary and verify hidden node is demoted"); + rst.restart(regularSecondary); + rst.awaitSecondaryNodes(30000, [regularSecondary]); + + // Wait for topology update + assert.soon(() => { + const status = assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})); + for (let member of status.members) { + if (member.name === regularSecondary.host) { + return member.health === 1 && member.state === 2; + } + } + return false; + }, "Secondary should be healthy", 30000); + + sleep(3000); + + const helloResp2 = assert.commandWorked(primary.adminCommand({hello: 1})); + hiddenPromoted = helloResp2.hosts && helloResp2.hosts.includes(hiddenNode.host); + assert.eq(false, hiddenPromoted, + "Hidden node should be removed from hosts when secondary is healthy again"); + + rst.stopSet(); + + jsTestLog("[PASSED] CASE 1: 3-member replica set test completed successfully"); +})(); + +// ============================================================================= +// CASE 2: Sharded cluster with autoPromoteHidden +// ============================================================================= +(function testCase2_ShardedCluster() { + jsTestLog("===================================================================="); + jsTestLog("CASE 2: Testing sharded cluster with autoPromoteHidden"); + jsTestLog("===================================================================="); + + const st = new ShardingTest({ + shards: 1, + rs0: { + nodes: [ + {}, // Primary + {}, // Secondary 1 + {rsConfig: {priority: 0, hidden: true}} // Hidden node + ], + settings: {heartbeatIntervalMillis: 500} + }, + config: 1, + mongos: 1 + }); + + const shard0Rst = st.rs0; + const primary = shard0Rst.getPrimary(); + const secondaries = shard0Rst.getSecondaries(); + const regularSecondary = secondaries[0]; + const hiddenNode = secondaries[1]; + + jsTestLog("Setting up sharded collection with test data..."); + const mongos = st.s; + assert.commandWorked(mongos.adminCommand({enableSharding: dbName})); + assert.commandWorked(mongos.adminCommand({ + shardCollection: dbName + "." + collName, + key: {_id: 1} + })); + + assert.commandWorked(mongos.getDB(dbName).getCollection(collName).insert( + [{_id: 1, data: "shard_test"}] + )); + shard0Rst.awaitReplication(); + + jsTestLog("Step 2.1: Test reads through mongos with secondary read preference"); + // This should work - mongos can route to the non-hidden secondary + const readResult1 = assert.commandWorked(mongos.getDB(dbName).runCommand({ + find: collName, + $readPreference: {mode: "secondary"} + })); + assert.eq(1, readResult1.cursor.firstBatch.length, "Should read from secondary"); + jsTestLog("[PASS] Read from secondary through mongos succeeded"); + + jsTestLog("Step 2.2: Stop the non-hidden secondary on shard"); + shard0Rst.stop(regularSecondary); + + // Wait for detection + assert.soon(() => { + const status = assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})); + for (let member of status.members) { + if (member.name === regularSecondary.host) { + return member.health === 0; + } + } + return false; + }, "Shard primary should detect secondary is down", 30000); + + sleep(3000); + + jsTestLog("Step 2.3: Test read with secondary preference should fail (no visible secondary)"); + // Without autoPromoteHidden, mongos cannot route to hidden node + try { + mongos.getDB(dbName).getMongo().setReadPref("secondary"); + const readResult2 = mongos.getDB(dbName).getCollection(collName).find().limit(1).toArray(); + // This might timeout or fail because no secondary is available + jsTestLog("Read attempt completed (may have retried to primary): " + tojson(readResult2)); + } catch (e) { + jsTestLog("[PASS] Read with secondary preference failed as expected: " + e.message); + } + + jsTestLog("Step 2.4: Enable autoPromoteHidden on the shard"); + let config = shard0Rst.getReplSetConfigFromNode(); + config.version++; + config.settings = config.settings || {}; + config.settings.autoPromoteHidden = true; + + assert.commandWorked(primary.adminCommand({replSetReconfig: config})); + + sleep(5000); + + // Verify hidden node is promoted + const helloResp = assert.commandWorked(primary.adminCommand({hello: 1})); + assert(helloResp.hosts.includes(hiddenNode.host), + "Hidden node should be in hosts after autoPromoteHidden"); + jsTestLog("[PASS] Hidden node promoted in shard replica set"); + + jsTestLog("Step 2.5: Test read with secondary preference should now work"); + // Mongos should now be able to route to the promoted hidden node + mongos.getDB(dbName).getMongo().setReadPref("secondary"); + assert.soon(() => { + try { + const readResult3 = mongos.getDB(dbName).getCollection(collName).find().limit(1).toArray(); + return readResult3.length === 1; + } catch (e) { + jsTestLog("Read still failing, retrying: " + e.message); + return false; + } + }, "Should be able to read from promoted hidden node", 30000); + jsTestLog("[PASS] Read with secondary preference succeeded after autoPromoteHidden"); + + jsTestLog("Step 2.6: Restart secondary and verify hidden node is demoted"); + shard0Rst.restart(regularSecondary); + shard0Rst.awaitSecondaryNodes(30000, [regularSecondary]); + + sleep(5000); + + const helloResp2 = assert.commandWorked(primary.adminCommand({hello: 1})); + assert(!helloResp2.hosts.includes(hiddenNode.host), + "Hidden node should be demoted when secondary is healthy"); + + st.stop(); + + jsTestLog("[PASSED] CASE 2: Sharded cluster test completed successfully"); +})(); + +// ============================================================================= +// CASE 3: 5-member replica set with 2 hidden nodes +// ============================================================================= +(function testCase3_FiveNodeReplSet() { + jsTestLog("===================================================================="); + jsTestLog("CASE 3: Testing 5-member replica set with 2 hidden nodes"); + jsTestLog("===================================================================="); + + const rst = new ReplSetTest({ + name: "auto_promote_hidden_case3", + nodes: [ + {}, // Primary + {}, // Secondary 1 + {}, // Secondary 2 + {rsConfig: {priority: 0, hidden: true}}, // Hidden node 1 + {rsConfig: {priority: 0, hidden: true}} // Hidden node 2 + ], + settings: {heartbeatIntervalMillis: 500, electionTimeoutMillis: 2000} + }); + + rst.startSet(); + rst.initiate(); + + const primary = rst.getPrimary(); + const allSecondaries = rst.getSecondaries(); + + // Identify non-hidden and hidden secondaries + const configMembers = rst.getReplSetConfigFromNode().members; + const regularSecondaries = []; + const hiddenNodes = []; + + for (let node of allSecondaries) { + const memberConfig = configMembers.find(m => m.host === node.host); + if (memberConfig.hidden) { + hiddenNodes.push(node); + } else { + regularSecondaries.push(node); + } + } + + jsTestLog("Regular secondaries: " + regularSecondaries.map(n => n.host)); + jsTestLog("Hidden nodes: " + hiddenNodes.map(n => n.host)); + + assert.eq(2, regularSecondaries.length, "Should have 2 regular secondaries"); + assert.eq(2, hiddenNodes.length, "Should have 2 hidden nodes"); + + jsTestLog("Setting up test data..."); + assert.commandWorked(primary.getDB(dbName).getCollection(collName).insert( + [{_id: 1, data: "test"}, {_id: 2, data: "test2"}, {_id: 3, data: "test3"}] + )); + rst.awaitReplication(); + + jsTestLog("Step 3.1: Verify initial reads work with secondary preference"); + testRead(primary, dbName, collName, {mode: "secondary"}, true, + "Read with secondary preference (2 healthy secondaries)"); + + jsTestLog("Step 3.2: Stop BOTH non-hidden secondaries"); + rst.stop(regularSecondaries[0]); + rst.stop(regularSecondaries[1]); + + // Wait for primary to detect both secondaries are down + assert.soon(() => { + const status = assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})); + let downCount = 0; + for (let member of status.members) { + if ((member.name === regularSecondaries[0].host || + member.name === regularSecondaries[1].host) && + member.health === 0) { + downCount++; + } + } + return downCount === 2; + }, "Primary should detect both secondaries are down", 30000); + + sleep(3000); + + jsTestLog("Step 3.3: Test read preferences WITHOUT autoPromoteHidden"); + // Primary read should work + testRead(primary, dbName, collName, {mode: "primary"}, true, + "Read with primary preference (both secondaries down)"); + + // Secondary read should fail - no visible secondary + testRead(primary, dbName, collName, {mode: "secondary"}, false, + "Read with secondary preference (both secondaries down, autoPromote=false)"); + + // Verify hidden nodes are NOT in hosts list + let helloResp = assert.commandWorked(primary.adminCommand({hello: 1})); + jsTestLog("Hello response before autoPromoteHidden: " + tojson(helloResp.hosts)); + + let hiddenCount = 0; + for (let host of (helloResp.hosts || [])) { + if (host === hiddenNodes[0].host || host === hiddenNodes[1].host) { + hiddenCount++; + } + } + assert.eq(0, hiddenCount, "No hidden nodes should be in hosts list initially"); + + jsTestLog("Step 3.4: Enable autoPromoteHidden"); + let config = rst.getReplSetConfigFromNode(); + config.version++; + config.settings = config.settings || {}; + config.settings.autoPromoteHidden = true; + + assert.commandWorked(primary.adminCommand({replSetReconfig: config})); + + // Wait for topology update + sleep(5000); + + jsTestLog("Step 3.5: Verify hidden nodes are promoted to hosts list"); + helloResp = assert.commandWorked(primary.adminCommand({hello: 1})); + jsTestLog("Hello response after autoPromoteHidden: " + tojson(helloResp.hosts)); + + hiddenCount = 0; + for (let host of (helloResp.hosts || [])) { + if (host === hiddenNodes[0].host || host === hiddenNodes[1].host) { + hiddenCount++; + } + } + assert.gte(hiddenCount, 1, "At least one hidden node should be promoted to hosts list"); + jsTestLog("[PASS] " + hiddenCount + " hidden node(s) promoted to hosts list"); + + jsTestLog("Step 3.6: Test read with secondary preference - should now succeed"); + testRead(primary, dbName, collName, {mode: "secondary"}, true, + "Read with secondary preference (autoPromote=true, reading from hidden)"); + + // Verify we can actually read from hidden nodes + assert.commandWorked(hiddenNodes[0].getDB(dbName).runCommand({ + find: collName, + limit: 1 + })); + jsTestLog("[PASS] Direct read from hidden node succeeded"); + + jsTestLog("Step 3.7: Test secondaryPreferred - should route to hidden"); + testRead(primary, dbName, collName, {mode: "secondaryPreferred"}, true, + "Read with secondaryPreferred (should route to promoted hidden)"); + + jsTestLog("Step 3.8: Restart one non-hidden secondary"); + rst.restart(regularSecondaries[0]); + rst.awaitSecondaryNodes(30000, [regularSecondaries[0]]); + + // Wait for primary to detect the secondary is healthy + assert.soon(() => { + const status = assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})); + for (let member of status.members) { + if (member.name === regularSecondaries[0].host) { + return member.health === 1 && member.state === 2; + } + } + return false; + }, "First secondary should be healthy", 30000); + + sleep(3000); + + jsTestLog("Step 3.9: Verify hidden nodes are demoted (one healthy secondary is enough)"); + helloResp = assert.commandWorked(primary.adminCommand({hello: 1})); + jsTestLog("Hello response after secondary restart: " + tojson(helloResp.hosts)); + + hiddenCount = 0; + for (let host of (helloResp.hosts || [])) { + if (host === hiddenNodes[0].host || host === hiddenNodes[1].host) { + hiddenCount++; + } + } + assert.eq(0, hiddenCount, + "Hidden nodes should be demoted when at least one non-hidden secondary is healthy"); + + jsTestLog("Step 3.10: Verify reads still work from regular secondary"); + testRead(primary, dbName, collName, {mode: "secondary"}, true, + "Read with secondary preference (after hidden demotion)"); + + // Restart the other secondary for cleanup + rst.restart(regularSecondaries[1]); + + rst.stopSet(); + + jsTestLog("[PASSED] CASE 3: 5-member replica set test completed successfully"); +})(); + +// ============================================================================= +// Additional verification: Read preference behavior +// ============================================================================= +(function testCase1Extended_ReadPreferenceVariations() { + jsTestLog("===================================================================="); + jsTestLog("EXTENDED: Testing various read preference modes"); + jsTestLog("===================================================================="); + + const rst = new ReplSetTest({ + name: "auto_promote_hidden_extended", + nodes: [ + {}, + {}, + {rsConfig: {priority: 0, hidden: true}} + ], + settings: {heartbeatIntervalMillis: 500} + }); + + rst.startSet(); + rst.initiate(); + + const primary = rst.getPrimary(); + const secondary = rst.getSecondary(); + + // Enable autoPromoteHidden from the start + let config = rst.getReplSetConfigFromNode(); + config.version++; + config.settings = config.settings || {}; + config.settings.autoPromoteHidden = true; + assert.commandWorked(primary.adminCommand({replSetReconfig: config})); + + // Insert test data + assert.commandWorked(primary.getDB(dbName).getCollection(collName).insert( + [{_id: 1, x: 1}] + )); + rst.awaitReplication(); + + jsTestLog("Step EXT.1: Test all read preference modes with healthy secondary"); + testRead(primary, dbName, collName, {mode: "primary"}, true, "primary mode"); + testRead(primary, dbName, collName, {mode: "primaryPreferred"}, true, "primaryPreferred mode"); + testRead(primary, dbName, collName, {mode: "secondary"}, true, "secondary mode"); + testRead(primary, dbName, collName, {mode: "secondaryPreferred"}, true, "secondaryPreferred mode"); + testRead(primary, dbName, collName, {mode: "nearest"}, true, "nearest mode"); + + jsTestLog("Step EXT.2: Stop secondary and retest"); + rst.stop(secondary); + + assert.soon(() => { + const status = assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})); + for (let member of status.members) { + if (member.name === secondary.host) { + return member.health === 0; + } + } + return false; + }, "Secondary should be down", 30000); + + sleep(5000); // Wait for promotion + + jsTestLog("Step EXT.3: Test all read preference modes with promoted hidden node"); + testRead(primary, dbName, collName, {mode: "primary"}, true, + "primary mode (hidden promoted)"); + testRead(primary, dbName, collName, {mode: "primaryPreferred"}, true, + "primaryPreferred mode (hidden promoted)"); + testRead(primary, dbName, collName, {mode: "secondary"}, true, + "secondary mode (should route to promoted hidden)"); + testRead(primary, dbName, collName, {mode: "secondaryPreferred"}, true, + "secondaryPreferred mode (hidden promoted)"); + testRead(primary, dbName, collName, {mode: "nearest"}, true, + "nearest mode (hidden promoted)"); + + rst.restart(secondary); + rst.stopSet(); + + jsTestLog("[PASSED] EXTENDED TESTS: All read preference modes work correctly"); +})(); + +jsTestLog("========================================================================"); +jsTestLog("ALL TEST CASES PASSED!"); +jsTestLog("========================================================================"); +jsTestLog("Summary:"); +jsTestLog(" [PASSED] Case 1: 3-member replica set"); +jsTestLog(" [PASSED] Case 2: Sharded cluster"); +jsTestLog(" [PASSED] Case 3: 5-member replica set with 2 hidden nodes"); +jsTestLog(" [PASSED] Extended: Various read preference modes"); +jsTestLog("========================================================================"); + diff --git a/src/mongo/db/repl/repl_set_config.h b/src/mongo/db/repl/repl_set_config.h index eb29db24faa9f..240a2204cc73a 100644 --- a/src/mongo/db/repl/repl_set_config.h +++ b/src/mongo/db/repl/repl_set_config.h @@ -439,6 +439,14 @@ class ReplSetConfig : private MutableReplSetConfig { return getSettings()->getChainingAllowed(); } + /** + * Returns true if hidden nodes should be automatically promoted when all non-hidden + * secondary nodes are unhealthy. + */ + bool getAutoPromoteHidden() const { + return getSettings()->getAutoPromoteHidden(); + } + /** * Returns whether all members of this replica set have hostname localhost. */ diff --git a/src/mongo/db/repl/repl_set_config.idl b/src/mongo/db/repl/repl_set_config.idl index d04384771c950..32bb1d9760111 100644 --- a/src/mongo/db/repl/repl_set_config.idl +++ b/src/mongo/db/repl/repl_set_config.idl @@ -137,6 +137,11 @@ structs: type: objectid optional: true validator: { callback: "validateReplicaSetIdNotNull"} + autoPromoteHidden: + type: safeBool + default: false + description: "When true, allows hidden nodes to be automatically promoted to appear + in the hosts list when all non-hidden secondary nodes are unhealthy" ReplSetConfigBase: description: "The complete configuration for the replica set" diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp index 26fce4e870341..3cdbb41c3c0da 100644 --- a/src/mongo/db/repl/topology_coordinator.cpp +++ b/src/mongo/db/repl/topology_coordinator.cpp @@ -2321,13 +2321,43 @@ void TopologyCoordinator::fillHelloForReplSet(std::shared_ptr res invariant(!_rsConfig.members().empty()); + bool promoteHidden = true; + if (_rsConfig.getAutoPromoteHidden()) { + for (int i = 0; i < _rsConfig.getNumMembers(); i++) { + const auto& memberCfg = _rsConfig.getMemberAt(i); + if (memberCfg.isHidden()) { + // Only consider non-hidden secondaries for health check + continue; + } + const auto& memberData = _memberData.at(i); + // Check if this is a healthy secondary + bool isHealthySecondary = false; + if (i == _selfIndex) { + // For self, we are always "up", just check if we are secondary + isHealthySecondary = myState.secondary(); + } else { + // For other members, check both state and health from heartbeat data + isHealthySecondary = memberData.getState().secondary() && memberData.up(); + } + + if (isHealthySecondary) { + promoteHidden = false; + break; + } + } + } else { + promoteHidden = false; + } + for (const auto& member : _rsConfig.members()) { - if (member.isHidden() || member.getSecondaryDelay() > Seconds{0}) { + if (member.getSecondaryDelay() > Seconds{0}) { continue; } auto hostView = member.getHostAndPort(horizonString); - if (member.isElectable()) { + if (member.isHidden() && promoteHidden) { + response->addHost(std::move(hostView)); + } else if (member.isElectable()) { response->addHost(std::move(hostView)); } else if (member.isArbiter()) { response->addArbiter(std::move(hostView)); @@ -2359,7 +2389,7 @@ void TopologyCoordinator::fillHelloForReplSet(std::shared_ptr res if (selfConfig.getSecondaryDelay() > Seconds(0)) { response->setSecondaryDelaySecs(selfConfig.getSecondaryDelay()); } - if (selfConfig.isHidden()) { + if (selfConfig.isHidden() && !promoteHidden) { response->setIsHidden(true); } if (!selfConfig.shouldBuildIndexes()) {