From 51f91c3963f6e340de5298a690e87fe8ab5acda9 Mon Sep 17 00:00:00 2001 From: Siyang Tang Date: Sat, 1 Nov 2025 22:46:29 +0800 Subject: [PATCH] [fix](schema-change) Prevent coredump when reading non-overlapping segments from a single rowset during heavy schema change (#57191) ### What problem does this PR solve? Problem Summary: ``` *** Query id: 0-0 *** *** is nereids: 0 *** *** tablet id: 1751140494462 *** *** Aborted at 1751208065 (unix time) try "date -d @1751208065" if you are using GNU date *** *** Current BE git commitID: 34d8342994 *** *** SIGSEGV invalid permissions for mapped object (@0x556ef4dfc000) received by PID 4975 (TID 5587 OR 0x7f71da3c4640) from PID 18446744073522888704; stack trace: *** 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/common/signal_handler.h:421 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so 2# JVM_handle_linux_signal in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so 3# 0x00007F731FC1F520 in /lib/x86_64-linux-gnu/libc.so.6 4# doris::vectorized::VMergeIteratorContext::copy_rows(doris::vectorized::Block*, bool) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vgeneric_iterators.cpp:148 5# doris::Status doris::vectorized::VMergeIterator::_next_batch(doris::vectorized::Block*) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vgeneric_iterators.h:242 6# doris::vectorized::VMergeIterator::next_batch(doris::vectorized::Block*) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/vec/olap/vgeneric_iterators.h:201 7# doris::BetaRowsetReader::next_block(doris::vectorized::Block*) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/rowset/beta_rowset_reader.cpp:378 8# doris::VSchemaChangeDirectly::_inner_process(std::shared_ptr, doris::RowsetWriter*, std::shared_ptr, std::shared_ptr, std::shared_ptr) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/schema_change.cpp:523 9# doris::SchemaChange::process(std::shared_ptr, doris::RowsetWriter*, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/schema_change.h:120 10# doris::SchemaChangeJob::_convert_historical_rowsets(doris::SchemaChangeParams const&, long*) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/schema_change.cpp:1179 11# doris::SchemaChangeJob::_do_process_alter_tablet(doris::TAlterTabletReqV2 const&) in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be 12# doris::SchemaChangeJob::process_alter_tablet(doris::TAlterTabletReqV2 const&) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/olap/schema_change.cpp:759 13# doris::alter_tablet_callback(doris::StorageEngine&, doris::TAgentTaskRequest const&) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/agent/task_worker_pool.cpp:1990 14# std::_Function_handler(doris::TAgentTaskRequest const&) const::{lambda()#1}>::_M_invoke(std::_Any_data const&) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291 15# doris::ThreadPool::dispatch_thread() at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/util/threadpool.cpp:609 16# doris::Thread::supervise_thread(void*) at /home/zcp/repo_center/doris_branch-3.0/doris/be/src/util/thread.cpp:499 17# start_thread at ./nptl/pthread_create.c:442 18# 0x00007F731FD03850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83 ``` Schema of input block from base tablet should merge dropped column. --- be/src/olap/schema_change.cpp | 4 +- .../test_non_overlap_seg_heavy_sc.out | 7 ++ .../test_non_overlap_seg_heavy_sc.groovy | 74 +++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/schema_change_p0/test_non_overlap_seg_heavy_sc.out create mode 100644 regression-test/suites/schema_change_p0/test_non_overlap_seg_heavy_sc.groovy diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index cc1879fce88b43..7d4bc11366e791 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -552,7 +552,7 @@ Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader bool eof = false; do { auto new_block = vectorized::Block::create_unique(new_tablet_schema->create_block()); - auto ref_block = vectorized::Block::create_unique(base_tablet_schema->create_block()); + auto ref_block = vectorized::Block::create_unique(base_tablet_schema->create_block(false)); auto st = rowset_reader->next_block(ref_block.get()); if (!st) { @@ -621,7 +621,7 @@ Status VBaseSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset bool eof = false; do { - auto ref_block = vectorized::Block::create_unique(base_tablet_schema->create_block()); + auto ref_block = vectorized::Block::create_unique(base_tablet_schema->create_block(false)); auto st = rowset_reader->next_block(ref_block.get()); if (!st) { if (st.is()) { diff --git a/regression-test/data/schema_change_p0/test_non_overlap_seg_heavy_sc.out b/regression-test/data/schema_change_p0/test_non_overlap_seg_heavy_sc.out new file mode 100644 index 00000000000000..0780fb12bdbe65 --- /dev/null +++ b/regression-test/data/schema_change_p0/test_non_overlap_seg_heavy_sc.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +2 2 +3 3 +4 4 +5 5 + diff --git a/regression-test/suites/schema_change_p0/test_non_overlap_seg_heavy_sc.groovy b/regression-test/suites/schema_change_p0/test_non_overlap_seg_heavy_sc.groovy new file mode 100644 index 00000000000000..f95fbaf4297f24 --- /dev/null +++ b/regression-test/suites/schema_change_p0/test_non_overlap_seg_heavy_sc.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_non_overlap_seg_heavy_sc") { + def tblName = "test_non_overlap_seg_heavy_sc" + sql """ + DROP TABLE IF EXISTS ${tblName}_src + """ + sql """ + CREATE TABLE IF NOT EXISTS ${tblName}_src + ( + k INT NOT NULL, + v1 INT NOT NULL, + v2 INT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 5 + PROPERTIES( + "replication_num" = "1", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + + sql """ + DROP TABLE IF EXISTS ${tblName}_dst + """ + sql """ + CREATE TABLE IF NOT EXISTS ${tblName}_dst + ( + k INT NOT NULL, + v1 INT NOT NULL, + v2 INT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + + sql """ INSERT INTO ${tblName}_src VALUES (1, 1, 1),(2, 2, 2),(3, 3, 3),(4, 4, 4),(5, 5, 5) """ + + sql """ INSERT INTO ${tblName}_dst SELECT * FROM ${tblName}_src """ + + sql """ DELETE FROM ${tblName}_dst WHERE v1 = 1 """ + + sql """ ALTER TABLE ${tblName}_dst DROP COLUMN v1""" + + sql """ ALTER TABLE ${tblName}_dst MODIFY COLUMN v2 STRING NOT NULL """ + + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tblName}_dst' ORDER BY createtime DESC LIMIT 1 """ + time 600 + } + + qt_sql """ SELECT * FROM ${tblName}_dst ORDER BY k """ +}