branch-3.1: [fix](mysql) Fix SSL unwrap infinite loop on handshake failure #57364

fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java

@@ -597,7 +597,7 @@ private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLExce
             case OK:
                 return true;
             case CLOSED:
-                sslEngine.closeOutbound();
+                SslEngineHelper.checkClosedProgress("wrap", sslEngineResult, sslEngine, false);
                 return true;
             case BUFFER_OVERFLOW:
                 // Could attempt to drain the serverNetData buffer of any already obtained
@@ -615,13 +615,13 @@ private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLExce
         }
     }
 
-    private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) {
+    private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) throws SSLException {
         switch (sslEngineResult.getStatus()) {
             // normal status.
             case OK:
                 return true;
             case CLOSED:
-                sslEngine.closeOutbound();
+                SslEngineHelper.checkClosedProgress("unwrap", sslEngineResult, sslEngine, true);
                 return true;
             case BUFFER_OVERFLOW:
                 // Could attempt to drain the clientAppData buffer of any already obtained

fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSslContext.java

@@ -227,14 +227,13 @@ private void handleNeedUnwrap(MysqlChannel channel) {
         }
     }
 
-
     private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLException {
         switch (sslEngineResult.getStatus()) {
             // normal status.
             case OK:
                 return true;
             case CLOSED:
-                sslEngine.closeOutbound();
+                SslEngineHelper.checkClosedProgress("wrap", sslEngineResult, sslEngine, false);
                 return true;
             case BUFFER_OVERFLOW:
                 // Could attempt to drain the serverNetData buffer of any already obtained
@@ -252,13 +251,13 @@ private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLExce
         }
     }
 
-    private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) {
+    private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) throws SSLException {
         switch (sslEngineResult.getStatus()) {
             // normal status.
             case OK:
                 return true;
             case CLOSED:
-                sslEngine.closeOutbound();
+                SslEngineHelper.checkClosedProgress("unwrap", sslEngineResult, sslEngine, true);
                 return true;
             case BUFFER_OVERFLOW:
                 // Could attempt to drain the clientAppData buffer of any already obtained

fe/fe-core/src/main/java/org/apache/doris/mysql/SslEngineHelper.java

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mysql;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLException;
+
+/**
+ * Helper class for SSL engine operations.
+ */
+public class SslEngineHelper {
+    private static final Logger LOG = LogManager.getLogger(SslEngineHelper.class);
+
+    /**
+     * Check if SSL engine operation has made progress when closed.
+     * @param operation operation name for logging ("wrap" or "unwrap")
+     * @param sslEngineResult the SSL engine result to check
+     * @param sslEngine the SSL engine instance
+     * @param closeInbound whether to close inbound (true for unwrap, false for wrap)
+     * @throws SSLException if no progress was made
+     */
+    public static void checkClosedProgress(String operation, SSLEngineResult sslEngineResult,
+                                           SSLEngine sslEngine, boolean closeInbound) throws SSLException {
+        int consumed = sslEngineResult.bytesConsumed();
+        int produced = sslEngineResult.bytesProduced();
+        if (consumed == 0 && produced == 0) {
+            LOG.warn("SSLEngine {} closed with no progress. status={}, handshake={}, "
+                    + "bytesConsumed={}, bytesProduced={}", operation,
+                    sslEngineResult.getStatus(), sslEngineResult.getHandshakeStatus(),
+                    consumed, produced);
+            if (closeInbound) {
+                try {
+                    sslEngine.closeInbound();
+                } catch (SSLException e) {
+                    LOG.warn("Error when closing SSL inbound during " + operation, e);
+                }
+            }
+            sslEngine.closeOutbound();
+            throw new SSLException("SSL " + operation + " closed with no progress (handshakeStatus="
+                    + sslEngineResult.getHandshakeStatus() + ", bytesConsumed="
+                    + consumed + ", bytesProduced=" + produced + ")");
+        }
+        if (closeInbound) {
+            try {
+                sslEngine.closeInbound();
+            } catch (SSLException e) {
+                LOG.debug("closeInbound on normal " + operation + " close failed", e);
+            }
+        }
+        LOG.debug("SSLEngine {} closed normally. status={}, handshake={}, "
+                + "bytesConsumed={}, bytesProduced={}", operation,
+                sslEngineResult.getStatus(), sslEngineResult.getHandshakeStatus(),
+                consumed, produced);
+        sslEngine.closeOutbound();
+    }
+}