diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 22cf6a083bbbfa..d70fd41beae611 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -523,13 +523,18 @@ void TabletMeta::init_schema_from_thrift(const TTabletSchema& tablet_schema, } else { unique_id = col_ordinal_to_unique_id.at(col_ordinal); } - col_ordinal++; init_column_from_tcolumn(unique_id, tcolumn, column); - if (column->is_bf_column()) { has_bf_columns = true; } + if (column->name() == BINLOG_LSN_COL) { + tablet_schema_pb->set_binlog_lsn_col_idx(col_ordinal); + } else if (column->name() == BINLOG_TIMESTAMP_COL) { + tablet_schema_pb->set_binlog_timestamp_col_idx(col_ordinal); + } + col_ordinal++; + if (tablet_schema.__isset.indexes) { for (auto& index : tablet_schema.indexes) { if (index.index_type == TIndexType::type::BLOOMFILTER || diff --git a/be/src/storage/utils.h b/be/src/storage/utils.h index 342e04ed4717e6..f1dc59757bd196 100644 --- a/be/src/storage/utils.h +++ b/be/src/storage/utils.h @@ -41,6 +41,7 @@ static const std::string SKIP_BITMAP_COL = "__DORIS_SKIP_BITMAP_COL__"; static const std::string SEQUENCE_COL = "__DORIS_SEQUENCE_COL__"; static const std::string BINLOG_TIMESTAMP_COL = "__DORIS_BINLOG_TIMESTAMP__"; static const std::string BINLOG_LSN_COL = "__DORIS_BINLOG_LSN__"; +static const std::string BINLOG_OP_COL = "__DORIS_BINLOG_OP__"; // 用来加速运算 const static int32_t g_power_table[] = {1, 10, 100, 1000, 10000, diff --git a/be/test/storage/tablet/tablet_meta_test.cpp b/be/test/storage/tablet/tablet_meta_test.cpp index 33b784f666f1e0..6073c5592a8471 100644 --- a/be/test/storage/tablet/tablet_meta_test.cpp +++ b/be/test/storage/tablet/tablet_meta_test.cpp @@ -28,6 +28,8 @@ #include "storage/file_header.h" #include "storage/rowset/rowset.h" #include "storage/tablet/tablet_schema.h" +#include "storage/utils.h" +#include "testutil/creators.h" #include "testutil/mock_rowset.h" namespace doris { @@ -403,4 +405,98 @@ TEST(TabletMetaTest, TestDeleteBitmap) { EXPECT_EQ(d.cardinality(), 500); } +TEST(TabletMetaTest, test_row_binlog_schema_binlog_col_idx) { + // 1. Create TCreateTabletReq + TCreateTabletReq request; + request.tablet_id = 10086; + + // Manually construct the base tablet_schema + TTabletSchema base_schema; + base_schema.keys_type = TKeysType::UNIQUE_KEYS; + base_schema.short_key_column_count = 1; + + // Add normal column k1 + TColumn k1_col; + k1_col.column_name = "k1"; + TColumnType k1_type; + k1_type.type = TPrimitiveType::INT; + k1_col.column_type = k1_type; + k1_col.is_key = true; + k1_col.is_allow_null = false; + base_schema.columns.push_back(k1_col); + + // Add normal column v1 + TColumn v1_col; + v1_col.column_name = "v1"; + TColumnType v1_type; + v1_type.type = TPrimitiveType::INT; + v1_col.column_type = v1_type; + v1_col.is_key = false; + v1_col.is_allow_null = true; + v1_col.aggregation_type = TAggregationType::NONE; + base_schema.columns.push_back(v1_col); + + request.tablet_schema = base_schema; + + // Enable row binlog + testutil::enable_row_binlog(&request); + + // Get the full schema including binlog columns + TTabletSchema row_binlog_schema = request.row_binlog_schema; + + // Dynamically find the actual indices of binlog columns + int expected_lsn_idx = -1; + int expected_ts_idx = -1; + for (size_t i = 0; i < row_binlog_schema.columns.size(); ++i) { + if (row_binlog_schema.columns[i].column_name == BINLOG_LSN_COL) { + expected_lsn_idx = static_cast(i); + } + if (row_binlog_schema.columns[i].column_name == BINLOG_TIMESTAMP_COL) { + expected_ts_idx = static_cast(i); + } + } + + // Ensure binlog columns are found + ASSERT_NE(expected_lsn_idx, -1) << "Failed to find " << BINLOG_LSN_COL; + ASSERT_NE(expected_ts_idx, -1) << "Failed to find " << BINLOG_TIMESTAMP_COL; + + // Set col_unique_id + for (size_t i = 0; i < row_binlog_schema.columns.size(); ++i) { + row_binlog_schema.columns[i].col_unique_id = static_cast(i); + } + + // Build col_ordinal_to_unique_id mapping + std::unordered_map col_ordinal_to_unique_id; + for (uint32_t i = 0; i < row_binlog_schema.columns.size(); ++i) { + col_ordinal_to_unique_id[i] = i; + } + + // Construct SchemaCreateOptions + TabletMeta::SchemaCreateOptions options = { + .col_ordinal_to_unique_id = col_ordinal_to_unique_id, + .compression_type = TCompressionType::LZ4F, + .inverted_index_file_storage_format = TInvertedIndexFileStorageFormat::V2, + .next_unique_id = static_cast(row_binlog_schema.columns.size())}; + + // Call the function under test + TabletSchemaPB schema_pb; + TabletMeta::init_schema_from_thrift(row_binlog_schema, options, &schema_pb); + + // Verify binlog column indices match the dynamically found values + EXPECT_EQ(schema_pb.binlog_lsn_col_idx(), expected_lsn_idx); + EXPECT_EQ(schema_pb.binlog_timestamp_col_idx(), expected_ts_idx); + + // Create TabletSchema object from PB + TabletSchemaSPtr tablet_schema_ptr = std::make_shared(); + tablet_schema_ptr->init_from_pb(schema_pb); + + // Verify indices are correct after deserialization + EXPECT_EQ(tablet_schema_ptr->binlog_lsn_col_idx(), expected_lsn_idx); + EXPECT_EQ(tablet_schema_ptr->binlog_timestamp_col_idx(), expected_ts_idx); + + // Verify the column names at the found indices + EXPECT_EQ(tablet_schema_ptr->column(expected_lsn_idx).name(), BINLOG_LSN_COL); + EXPECT_EQ(tablet_schema_ptr->column(expected_ts_idx).name(), BINLOG_TIMESTAMP_COL); +} + } // namespace doris diff --git a/be/test/testutil/creators.h b/be/test/testutil/creators.h index 3034a01b086d1c..1979e50100b275 100644 --- a/be/test/testutil/creators.h +++ b/be/test/testutil/creators.h @@ -38,6 +38,7 @@ #include "runtime/query_context.h" #include "storage/binlog.h" #include "storage/tablet_info.h" +#include "storage/utils.h" #include "util/uid_util.h" namespace doris { @@ -220,6 +221,12 @@ inline void enable_row_binlog(TCreateTabletReq* request, int32_t row_binlog_sche row_binlog_schema.columns.push_back( create_tablet_column({std::string(kRowBinlogTimestampColName), TPrimitiveType::BIGINT, false, true, TAggregationType::NONE})); + for (auto& col : row_binlog_schema.columns) { + if (col.column_name == BINLOG_LSN_COL || col.column_name == BINLOG_OP_COL || + col.column_name == BINLOG_TIMESTAMP_COL) { + col.__set_is_allow_null(true); + } + } request->__set_row_binlog_schema(row_binlog_schema); }