From 547bae0cd215eeb00eecbba44ec2cceb967e73cc Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Sat, 1 Nov 2025 21:41:55 +0800 Subject: [PATCH] [fix](mysql) Fix SSL unwrap infinite loop on handshake failure (#57364) --- .../org/apache/doris/mysql/MysqlChannel.java | 6 +- .../apache/doris/mysql/MysqlSslContext.java | 7 +- .../apache/doris/mysql/SslEngineHelper.java | 75 +++++++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/SslEngineHelper.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java index 34494aa4296468..87a9ac83d881ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java @@ -597,7 +597,7 @@ private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLExce case OK: return true; case CLOSED: - sslEngine.closeOutbound(); + SslEngineHelper.checkClosedProgress("wrap", sslEngineResult, sslEngine, false); return true; case BUFFER_OVERFLOW: // Could attempt to drain the serverNetData buffer of any already obtained @@ -615,13 +615,13 @@ private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLExce } } - private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) { + private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) throws SSLException { switch (sslEngineResult.getStatus()) { // normal status. case OK: return true; case CLOSED: - sslEngine.closeOutbound(); + SslEngineHelper.checkClosedProgress("unwrap", sslEngineResult, sslEngine, true); return true; case BUFFER_OVERFLOW: // Could attempt to drain the clientAppData buffer of any already obtained diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSslContext.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSslContext.java index b59b493ceaf46e..6283e795cfebed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSslContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSslContext.java @@ -227,14 +227,13 @@ private void handleNeedUnwrap(MysqlChannel channel) { } } - private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLException { switch (sslEngineResult.getStatus()) { // normal status. case OK: return true; case CLOSED: - sslEngine.closeOutbound(); + SslEngineHelper.checkClosedProgress("wrap", sslEngineResult, sslEngine, false); return true; case BUFFER_OVERFLOW: // Could attempt to drain the serverNetData buffer of any already obtained @@ -252,13 +251,13 @@ private boolean handleWrapResult(SSLEngineResult sslEngineResult) throws SSLExce } } - private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) { + private boolean handleUnwrapResult(SSLEngineResult sslEngineResult) throws SSLException { switch (sslEngineResult.getStatus()) { // normal status. case OK: return true; case CLOSED: - sslEngine.closeOutbound(); + SslEngineHelper.checkClosedProgress("unwrap", sslEngineResult, sslEngine, true); return true; case BUFFER_OVERFLOW: // Could attempt to drain the clientAppData buffer of any already obtained diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/SslEngineHelper.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/SslEngineHelper.java new file mode 100644 index 00000000000000..816088d1f2a9ff --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/SslEngineHelper.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLEngineResult; +import javax.net.ssl.SSLException; + +/** + * Helper class for SSL engine operations. + */ +public class SslEngineHelper { + private static final Logger LOG = LogManager.getLogger(SslEngineHelper.class); + + /** + * Check if SSL engine operation has made progress when closed. + * @param operation operation name for logging ("wrap" or "unwrap") + * @param sslEngineResult the SSL engine result to check + * @param sslEngine the SSL engine instance + * @param closeInbound whether to close inbound (true for unwrap, false for wrap) + * @throws SSLException if no progress was made + */ + public static void checkClosedProgress(String operation, SSLEngineResult sslEngineResult, + SSLEngine sslEngine, boolean closeInbound) throws SSLException { + int consumed = sslEngineResult.bytesConsumed(); + int produced = sslEngineResult.bytesProduced(); + if (consumed == 0 && produced == 0) { + LOG.warn("SSLEngine {} closed with no progress. status={}, handshake={}, " + + "bytesConsumed={}, bytesProduced={}", operation, + sslEngineResult.getStatus(), sslEngineResult.getHandshakeStatus(), + consumed, produced); + if (closeInbound) { + try { + sslEngine.closeInbound(); + } catch (SSLException e) { + LOG.warn("Error when closing SSL inbound during " + operation, e); + } + } + sslEngine.closeOutbound(); + throw new SSLException("SSL " + operation + " closed with no progress (handshakeStatus=" + + sslEngineResult.getHandshakeStatus() + ", bytesConsumed=" + + consumed + ", bytesProduced=" + produced + ")"); + } + if (closeInbound) { + try { + sslEngine.closeInbound(); + } catch (SSLException e) { + LOG.debug("closeInbound on normal " + operation + " close failed", e); + } + } + LOG.debug("SSLEngine {} closed normally. status={}, handshake={}, " + + "bytesConsumed={}, bytesProduced={}", operation, + sslEngineResult.getStatus(), sslEngineResult.getHandshakeStatus(), + consumed, produced); + sslEngine.closeOutbound(); + } +}