SERVER-85804 Support Proxy Protocol on Mongod (#31814)

Co-authored-by: Sara Golemon <[email protected]>
GitOrigin-RevId: ab2539a

Author: 2 people

buildscripts/resmokeconfig/fully_disabled_feature_flags.yml

@@ -17,5 +17,5 @@
 - featureFlagEgressGrpcForSearch
 - featureFlagTrackUnshardedCollectionsUponCreation
 - featureFlagTSBucketingParametersUnchanged
-- featureFlagMongogProxyProcolSupport
+- featureFlagMongodProxyProcolSupport
 - featureFlagShardAuthoritativeDbMetadata

jstests/noPassthrough/network/mongod_proxy_protocol.js

@@ -0,0 +1,102 @@
+/**
+ *  Verify mongod support proxy protocol connections.
+ * @tags: [
+ *   requires_fcv_81,
+ *    # TODO (SERVER-97257): Re-enable this test or add an explanation why it is incompatible.
+ *    embedded_router_incompatible,
+ *    grpc_incompatible,
+ * ]
+ */
+
+if (_isWindows()) {
+    quit();
+}
+import {ProxyProtocolServer} from "jstests/sharding/libs/proxy_protocol.js";
+import {ReplSetTest} from "jstests/libs/replsettest.js";
+
+function runHello(port, loadBalanced) {
+    let uri = `mongodb://127.0.0.1:${port}`;
+    if (typeof loadBalanced != 'undefined') {
+        uri += `/?loadBalanced=${loadBalanced}`;
+    }
+    const conn = new Mongo(uri);
+    assert.neq(null, conn, 'Client was unable to connect to the load balancer port');
+    assert.commandWorked(conn.getDB('admin').runCommand({hello: 1}));
+}
+
+function failInvalidProtocol(node, port, id, attrs, loadBalanced, count) {
+    let uri = `mongodb://127.0.0.1:${port}`;
+    if (typeof loadBalanced != 'undefined') {
+        uri += `/?loadBalanced=${loadBalanced}`;
+    }
+    try {
+        new Mongo(uri);
+        assert(false, 'Client was unable to connect to the load balancer port');
+    } catch (err) {
+        assert(checkLog.checkContainsWithCountJson(node, id, attrs, count, undefined, true),
+               `Did not find log id ${tojson(id)} with attr ${tojson(attrs)} ${
+                   tojson(id)} times in the log`);
+    }
+}
+
+// Test that you can connect to the load balancer port over a proxy.
+function testProxyProtocolReplicaSet(ingressPort, egressPort, version) {
+    let proxy_server = new ProxyProtocolServer(ingressPort, egressPort, version);
+    proxy_server.start();
+
+    let rs = new ReplSetTest({nodes: 1, nodeOptions: {"proxyPort": egressPort}});
+    rs.startSet({setParameter: {featureFlagMongodProxyProcolSupport: true}});
+    rs.initiate();
+
+    // Connecting to the to the proxy port succeeds.
+    runHello(ingressPort, undefined);
+    runHello(ingressPort, false);
+
+    // Connecting to the to the proxy port with {loadBalanced: true} fails.
+    const lbmismatch = {
+        "error": "LoadBalancerSupportMismatch: Mongod does not support load-balanced connections"
+    };
+
+    const kCmdExecAssertion = 21962;
+    const node = rs.getPrimary();
+    failInvalidProtocol(node, ingressPort, kCmdExecAssertion, lbmismatch, "true", 1);
+
+    // Connecting to the standard port without proxy header succeeds.
+    const port = node.port;
+    runHello(port, undefined);
+    runHello(port, false);
+
+    // Connecting to the standard port without and with {loadBalanced:true} proxy header fails.
+    failInvalidProtocol(node, port, kCmdExecAssertion, lbmismatch, "true", 2);
+
+    // Connecting to the proxy port without proxy header fails.
+    const kProxyProtocolParseError = 6067900;
+    failInvalidProtocol(node, egressPort, kProxyProtocolParseError, undefined, "true", 1);
+    failInvalidProtocol(node, egressPort, kProxyProtocolParseError, undefined, "false", 2);
+    failInvalidProtocol(node, egressPort, kProxyProtocolParseError, undefined, undefined, 3);
+
+    proxy_server.stop();
+
+    // Connecting to the standard port with proxy header fails.
+    proxy_server = new ProxyProtocolServer(ingressPort, port, version);
+    proxy_server.start();
+    const attrs = {
+        "error": {
+            "code": ErrorCodes.OperationFailed,
+            "codeName": "OperationFailed",
+            "errmsg": "ProxyProtocol message detected on mongorpc port",
+        }
+    };
+    failInvalidProtocol(node, ingressPort, 22988, attrs, "true", 1);
+    failInvalidProtocol(node, ingressPort, 22988, attrs, "false", 2);
+    failInvalidProtocol(node, ingressPort, 22988, attrs, undefined, 3);
+    proxy_server.stop();
+
+    rs.stopSet();
+}
+
+const ingressPort = allocatePort();
+const egressPort = allocatePort();
+
+testProxyProtocolReplicaSet(ingressPort, egressPort, 1);
+testProxyProtocolReplicaSet(ingressPort, egressPort, 2);

src/mongo/db/mongod_main.cpp

@@ -188,6 +188,7 @@
 #include "mongo/db/s/sharding_initialization_mongod.h"
 #include "mongo/db/s/sharding_ready.h"
 #include "mongo/db/s/transaction_coordinator_service.h"
+#include "mongo/db/server_feature_flags_gen.h"
 #include "mongo/db/server_lifecycle_monitor.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/service_context.h"
@@ -322,7 +323,6 @@ const ntservice::NtServiceDefaultStrings defaultServiceStrings = {
 
 auto makeTransportLayer(ServiceContext* svcCtx) {
     boost::optional<int> routerPort;
-    boost::optional<int> loadBalancerPort;
     boost::optional<int> proxyPort;
 
     if (serverGlobalParams.routerPort) {
@@ -336,20 +336,23 @@ auto makeTransportLayer(ServiceContext* svcCtx) {
         // TODO SERVER-78730: add support for load-balanced connections.
     }
 
-    if (serverGlobalParams.proxyPort) {
-        proxyPort = *serverGlobalParams.proxyPort;
-        if (*proxyPort == serverGlobalParams.port) {
-            LOGV2_ERROR(9967800,
-                        "The proxy port must be different from the public listening port.",
-                        "port"_attr = serverGlobalParams.port);
-            quickExit(ExitCode::badOptions);
-        }
+    // (Ignore FCV check): The proxy port needs to be open before the FCV is set.
+    if (gFeatureFlagMongodProxyProcolSupport.isEnabledAndIgnoreFCVUnsafe()) {
+        if (serverGlobalParams.proxyPort) {
+            proxyPort = *serverGlobalParams.proxyPort;
+            if (*proxyPort == serverGlobalParams.port) {
+                LOGV2_ERROR(9967800,
+                            "The proxy port must be different from the public listening port.",
+                            "port"_attr = serverGlobalParams.port);
+                quickExit(ExitCode::badOptions);
+            }
 
-        if (routerPort && *proxyPort == *routerPort) {
-            LOGV2_ERROR(9967801,
-                        "The proxy port must be different from the public router port.",
-                        "port"_attr = *routerPort);
-            quickExit(ExitCode::badOptions);
+            if (routerPort && *proxyPort == *routerPort) {
+                LOGV2_ERROR(9967801,
+                            "The proxy port must be different from the public router port.",
+                            "port"_attr = *routerPort);
+                quickExit(ExitCode::badOptions);
+            }
         }
     }
 
@@ -369,11 +372,8 @@ auto makeTransportLayer(ServiceContext* svcCtx) {
     }
 #endif
 
-    return transport::TransportLayerManagerImpl::createWithConfig(&serverGlobalParams,
-                                                                  svcCtx,
-                                                                  useEgressGRPC,
-                                                                  std::move(loadBalancerPort),
-                                                                  std::move(routerPort));
+    return transport::TransportLayerManagerImpl::createWithConfig(
+        &serverGlobalParams, svcCtx, useEgressGRPC, std::move(proxyPort), std::move(routerPort));
 }
 
 ExitCode initializeTransportLayer(ServiceContext* serviceContext, BSONObjBuilder* timerReport) {

src/mongo/db/repl/replication_info.cpp

@@ -413,6 +413,10 @@ class CmdHello : public BasicCommandWithReplyBuilderInterface {
         auto cmd = idl::parseCommandDocument<HelloCommand>(
             IDLParserContext("hello", vts, dbName.tenantId(), sc), cmdObj);
 
+        uassert(ErrorCodes::LoadBalancerSupportMismatch,
+                "Mongod does not support load-balanced connections",
+                !cmd.getLoadBalanced().value_or(false));
+
         shardWaitInHello.execute(
             [&](const BSONObj& customArgs) { _handleHelloFailPoint(customArgs, opCtx, cmdObj); });
 

src/mongo/db/server_feature_flags.idl

@@ -134,9 +134,9 @@ feature_flags:
          cpp_varname: gFeatureFlagExposeClientIpInAuditLogs
          default: false
          shouldBeFCVGated: true
-    featureFlagMongogProxyProcolSupport:
+    featureFlagMongodProxyProcolSupport:
         description: "Enables non-OCS proxy protocol connections on Mongos and Mongod"
-        cpp_varname: gFeatureFlagMongogProxyProcolSupport
+        cpp_varname: gFeatureFlagMongodProxyProcolSupport
         default: false
         shouldBeFCVGated: true
     featureFlagRawDataCrudOperations:

src/mongo/transport/asio/asio_session_impl.cpp

@@ -747,6 +747,15 @@ Future<bool> CommonAsioSession::maybeHandshakeSSLForIngress(const MutableBufferS
     if (checkForHTTPRequest(buffer)) {
         return Future<bool>::makeReady(false);
     }
+
+    if (maybeProxyProtocolHeader(
+            StringData(asio::buffer_cast<const char*>(buffer), asio::buffer_size(buffer)))) {
+        // Protocol requirements mean that neither raw mongorpc nor TLS client hello will look like
+        // Proxy.
+        return Future<bool>::makeReady(
+            Status(ErrorCodes::OperationFailed, "ProxyProtocol message detected on mongorpc port"));
+    }
+
     // This logic was taken from the old mongo/util/net/sock.cpp.
     //
     // It lets us run both TLS and unencrypted mongo over the same port.

src/mongo/transport/proxy_protocol_header_parser.cpp

@@ -455,5 +455,8 @@ boost::optional<ParserResults> parseProxyProtocolHeader(StringData buffer) {
     }
 }
 
+bool maybeProxyProtocolHeader(StringData buffer) {
+    return buffer.startsWith(kV1Start) || buffer.startsWith(kV2Start);
+}
 
 }  // namespace mongo::transport

src/mongo/transport/proxy_protocol_header_parser.h

@@ -86,6 +86,16 @@ struct ParserResults {
  */
 boost::optional<ParserResults> parseProxyProtocolHeader(StringData buffer);
 
+/**
+ * Peek a buffer fo at least 12 bytes to determine if it may be a proxy protocol header.
+ *
+ * Note that this does not definitively identify the initial packet as proxy protocol,
+ * it only establishes that it is possible that it is such.
+ * To be used in determining appropriate error messages during otherwise failed
+ * initial handshakes only.
+ */
+bool maybeProxyProtocolHeader(StringData buffer);
+
 namespace proxy_protocol_details {
 static constexpr size_t kMaxUnixPathLength = 108;