Skip to content

Commit 8b1b089

Browse files
gouhongshengouhongshen
authored
to hotfix: fix clone indexes when alter table. (#23260)
### **User description** ## What type of PR is this? - [ ] API-change - [x] BUG - [ ] Improvement - [ ] Documentation - [ ] Feature - [ ] Test and CI - [ ] Code Refactoring ## Which issue(s) this PR fixes: issue ##23258 ## What this PR does / why we need it: Fixing the ALTER TABLE with a full-text index will cause index data duplication. ___ ### **PR Type** Bug fix ___ ### **Description** - Prevent index data duplication during ALTER TABLE operations - Skip inserting into indexes marked for cloning in copy operations - Add `skipIndexesCopy` parameter to track indexes being cloned - Add comprehensive test cases for ALTER TABLE with various index types ___ ### Diagram Walkthrough ```mermaid flowchart LR A["ALTER TABLE Operation"] --> B["Check skipIndexesCopy Map"] B --> C["Skip Insert for Cloned Indexes"] C --> D["Prevent Data Duplication"] E["AlterCopyOpt"] --> B ``` <details><summary><h3>File Walkthrough</h3></summary> <table><thead><tr><th></th><th align="left">Relevant files</th></tr></thead><tbody><tr><td><strong>Bug fix</strong></td><td><table> <tr> <td> <details> <summary><strong>build_dml_util.go</strong><dd><code>Add index cloning skip logic to prevent duplication</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> pkg/sql/plan/build_dml_util.go <ul><li>Added <code>skipIndexesCoyp</code> variable to track indexes being cloned during <br>ALTER TABLE<br> <li> Extract <code>SkipIndexesCopy</code> from <code>AlterCopyOpt</code> context when available<br> <li> Added logic to skip insert operations for indexes marked in <br><code>skipIndexesCopy</code> map<br> <li> Updated function signature of <code>buildInsertPlansWithRelatedHiddenTable</code> <br>to accept <code>skipIndexesCopy</code> parameter<br> <li> Updated all callers to pass the new <code>skipIndexesCopy</code> parameter</ul> </details> </td> <td><a href="https://github.com/matrixorigin/matrixone/pull/23260/files#diff-095fb233d51021791cb24454839b013236680bbc6bbc22e0d2f6741ac8fe7dff">+12/-3</a>&nbsp; &nbsp; </td> </tr> </table></td></tr><tr><td><strong>Tests</strong></td><td><table> <tr> <td> <details> <summary><strong>clone_in_alter_table_2.sql</strong><dd><code>Add test cases for ALTER TABLE index cloning</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> test/distributed/cases/snapshot/clone/clone_in_alter_table_2.sql <ul><li>New test file with comprehensive test cases for ALTER TABLE with <br>various index types<br> <li> Test case 1: FULLTEXT index with primary table and secondary key<br> <li> Test case 2: FULLTEXT index with multiple secondary keys<br> <li> Test case 3: Complex scenario with FULLTEXT, IVFFLAT, and HNSW vector <br>indexes<br> <li> Validates index table row counts to ensure no data duplication occurs</ul> </details> </td> <td><a href="https://github.com/matrixorigin/matrixone/pull/23260/files#diff-9c0c4513a70440bc95029f784a8b0cdc2fd63e654594507b43bfb36ce04cdac1">+163/-0</a>&nbsp; </td> </tr> <tr> <td> <details> <summary><strong>clone_in_alter_table_2.result</strong><dd><code>Add expected test results for ALTER TABLE</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary> <hr> test/distributed/cases/snapshot/clone/clone_in_alter_table_2.result <ul><li>Expected output results for the new test cases<br> <li> Validates correct row counts in index tables after ALTER TABLE <br>operations<br> <li> Confirms no data duplication in FULLTEXT, secondary, IVFFLAT, and HNSW <br>indexes</ul> </details> </td> <td><a href="https://github.com/matrixorigin/matrixone/pull/23260/files#diff-1c31cb9ccba87c68106869dbc154f5ac8f1a11056ad6a428071bc12c69fee79d">+149/-0</a>&nbsp; </td> </tr> </table></td></tr></tbody></table> </details> ___
1 parent 58944ac commit 8b1b089

3 files changed

Lines changed: 324 additions & 3 deletions

File tree

‎pkg/sql/plan/build_dml_util.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ func buildInsertPlans(
158158
ifNeedCheckPkDup := !builder.qry.LoadTag
159159
var indexSourceColTypes []*plan.Type
160160
var fuzzymessage *OriginTableMessageForFuzzy
161+
var skipIndexesCoyp map[string]bool
161162

162163
if v := builder.compCtx.GetContext().Value(defines.AlterCopyOpt{}); v != nil {
163164
dedupOpt := v.(*plan.AlterCopyOpt)
@@ -178,10 +179,13 @@ func buildInsertPlans(
178179
}
179180
}
180181
}
182+
skipIndexesCoyp = dedupOpt.SkipIndexesCopy
181183
}
182184
return buildInsertPlansWithRelatedHiddenTable(stmt, ctx, builder, insertBindCtx, objRef, tableDef,
183185
updateColLength, sourceStep, addAffectedRows, isFkRecursionCall, updatePkCol, pkFilterExpr,
184-
newPartitionExpr, ifExistAutoPkCol, ifNeedCheckPkDup, indexSourceColTypes, fuzzymessage, insertWithoutUniqueKeyMap, ifInsertFromUniqueColMap, nil)
186+
newPartitionExpr, ifExistAutoPkCol, ifNeedCheckPkDup, indexSourceColTypes, fuzzymessage,
187+
insertWithoutUniqueKeyMap, ifInsertFromUniqueColMap, nil, skipIndexesCoyp,
188+
)
185189
}
186190

187191
// buildUpdatePlans build update plan.
@@ -266,7 +270,7 @@ func buildUpdatePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC
266270
return buildInsertPlansWithRelatedHiddenTable(nil, ctx, builder, insertBindCtx, updatePlanCtx.objRef, updatePlanCtx.tableDef,
267271
updatePlanCtx.updateColLength, sourceStep, addAffectedRows, updatePlanCtx.isFkRecursionCall, updatePlanCtx.updatePkCol,
268272
updatePlanCtx.pkFilterExprs, partitionExpr, ifExistAutoPkCol, ifNeedCheckPkDup, indexSourceColTypes, fuzzymessage, nil, nil,
269-
updatePlanCtx.updateColPosMap)
273+
updatePlanCtx.updateColPosMap, nil)
270274
}
271275

272276
func getStepByNodeId(builder *QueryBuilder, nodeId int32) int {
@@ -853,7 +857,7 @@ func buildInsertPlansWithRelatedHiddenTable(
853857
updatePkCol bool, pkFilterExprs []*Expr, partitionExpr *Expr, ifExistAutoPkCol bool,
854858
checkInsertPkDupForHiddenIndexTable bool, indexSourceColTypes []*plan.Type, fuzzymessage *OriginTableMessageForFuzzy,
855859
insertWithoutUniqueKeyMap map[string]bool, ifInsertFromUniqueColMap map[string]bool,
856-
updateColPosMap map[string]int,
860+
updateColPosMap map[string]int, skipIndexesCopy map[string]bool,
857861
) error {
858862
//var lastNodeId int32
859863
var err error
@@ -870,6 +874,11 @@ func buildInsertPlansWithRelatedHiddenTable(
870874
continue
871875
}
872876

877+
// we will clone this index data to new index table, skip insert.
878+
if skipIndexesCopy != nil && skipIndexesCopy[indexdef.IndexName] {
879+
continue
880+
}
881+
873882
// append plan for the hidden tables of unique/secondary keys
874883
if indexdef.TableExist && catalog.IsRegularIndexAlgo(indexdef.IndexAlgo) {
875884
/********

‎test/distributed/cases/snapshot/clone/clone_in_alter_table_2.result‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
SET experimental_fulltext_index = 1;
2+
SET experimental_ivf_index = 1;
3+
SET experimental_hnsw_index = 1;
4+
drop database if exists db;
5+
create database db;
6+
use db;
7+
DROP TABLE IF EXISTS stress_alter_table;
8+
CREATE TABLE `stress_alter_table` (
9+
`md5_id` int NOT NULL,
10+
`b` varchar(65535) DEFAULT NULL,
11+
`delete_flag` int DEFAULT NULL,
12+
PRIMARY KEY (`md5_id`),
13+
FULLTEXT `fb`(`b`) WITH PARSER ngram,
14+
KEY `delete_flag_idx` (`delete_flag`)
15+
);
16+
INSERT INTO stress_alter_table VALUES (1, 'this is a one', 0);
17+
DELETE FROM `stress_alter_table` WHERE delete_flag = 0;
18+
INSERT INTO `stress_alter_table` (`md5_id`,`b`,`delete_flag`) VALUES (1,'this is a one',0);
19+
ALTER TABLE `stress_alter_table` MODIFY COLUMN delete_flag INT DEFAULT 0;
20+
select mo_ctl('dn','checkpoint','');
21+
mo_ctl(dn, checkpoint, )
22+
{\n "method": "Checkpoint",\n "result": [\n {\n "returnStr": "OK"\n }\n ]\n}\n
23+
SELECT COUNT(*) FROM `stress_alter_table`;
24+
COUNT(*)
25+
1
26+
UPDATE `stress_alter_table` SET delete_flag = 1 WHERE md5_id = '1';
27+
SELECT COUNT(*) FROM `stress_alter_table`;
28+
COUNT(*)
29+
1
30+
drop table stress_alter_table;
31+
create table t1 (id bigint primary key, delete_flag int, a int, body varchar(10), title text, FULLTEXT (title, body), key(delete_flag));
32+
insert into t1 values(1,1,1, "body", "title");
33+
ALTER TABLE t1 MODIFY COLUMN a int DEFAULT NULL;
34+
SET @inner_sql = (
35+
SELECT GROUP_CONCAT(
36+
DISTINCT CONCAT(
37+
'SELECT ''', mi.index_table_name,
38+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
39+
mi.index_table_name, '`'
40+
) SEPARATOR ' UNION ALL '
41+
)
42+
FROM mo_catalog.mo_indexes mi
43+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
44+
WHERE mt.relname = 't1'
45+
AND mt.reldatabase = 'db'
46+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
47+
AND mi.index_table_name IS NOT NULL
48+
AND mi.index_table_name != ''
49+
AND mi.column_name <> 'question'
50+
AND mi.column_name <> 'keyword'
51+
);
52+
SET @sql = CONCAT(
53+
'SELECT * FROM (',
54+
@inner_sql,
55+
') AS t ORDER BY cnt DESC'
56+
);
57+
PREPARE stmt FROM @sql;
58+
EXECUTE stmt;
59+
index_table_name cnt
60+
__mo_index_secondary_019b0be5-956f-77a1-b31b-b1db0d19540e 3
61+
__mo_index_secondary_019b0be5-956f-77a8-923c-2ce77693d21a 1
62+
DEALLOCATE PREPARE stmt;
63+
drop table t1;
64+
create table t2 (id int primary key, a int, b int, c int, key(a), key(b));
65+
insert into t2 values(1,1,1,1);
66+
ALTER TABLE t2 MODIFY COLUMN c int DEFAULT NULL;
67+
SET @inner_sql = (
68+
SELECT GROUP_CONCAT(
69+
DISTINCT CONCAT(
70+
'SELECT ''', mi.index_table_name,
71+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
72+
mi.index_table_name, '`'
73+
) SEPARATOR ' UNION ALL '
74+
)
75+
FROM mo_catalog.mo_indexes mi
76+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
77+
WHERE mt.relname = 't2'
78+
AND mt.reldatabase = 'db'
79+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
80+
AND mi.index_table_name IS NOT NULL
81+
AND mi.index_table_name != ''
82+
AND mi.column_name <> 'question'
83+
AND mi.column_name <> 'keyword'
84+
);
85+
SET @sql = CONCAT(
86+
'SELECT * FROM (',
87+
@inner_sql,
88+
') AS t ORDER BY cnt DESC'
89+
);
90+
PREPARE stmt FROM @sql;
91+
EXECUTE stmt;
92+
index_table_name cnt
93+
__mo_index_secondary_019b0be5-95ad-764d-b6f8-03e4c66741c5 1
94+
__mo_index_secondary_019b0be5-95ad-7645-8b2b-2cac0abec44d 1
95+
DEALLOCATE PREPARE stmt;
96+
drop table t2;
97+
create table t3 (
98+
id bigint primary key,
99+
delete_flag int,
100+
a vecf32(3),
101+
b vecf32(3),
102+
body varchar(10),
103+
title text,
104+
FULLTEXT (title, body),
105+
key(delete_flag),
106+
key ivf using ivfflat (a),
107+
key idx using hnsw(b) op_type 'vector_l2_ops'
108+
);
109+
insert into t3 values(1,1, '[0.1,0.2,0.3]','[0.1,0.2,0.3]', "body", "title");
110+
ALTER TABLE t3 MODIFY COLUMN delete_flag int DEFAULT NULL;
111+
SET @inner_sql = (
112+
SELECT GROUP_CONCAT(
113+
DISTINCT CONCAT(
114+
'SELECT ''', mi.index_table_name,
115+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
116+
mi.index_table_name, '`'
117+
) SEPARATOR ' UNION ALL '
118+
)
119+
FROM mo_catalog.mo_indexes mi
120+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
121+
WHERE mt.relname = 't3'
122+
AND mt.reldatabase = 'db'
123+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
124+
AND mi.index_table_name IS NOT NULL
125+
AND mi.index_table_name != ''
126+
AND mi.column_name <> 'question'
127+
AND mi.column_name <> 'keyword'
128+
);
129+
SET @sql = CONCAT(
130+
'SELECT * FROM (',
131+
@inner_sql,
132+
') AS t ORDER BY cnt DESC'
133+
);
134+
PREPARE stmt FROM @sql;
135+
EXECUTE stmt;
136+
index_table_name cnt
137+
__mo_index_secondary_019b0be5-95fd-7791-afd3-507749cef99c 7
138+
__mo_index_secondary_019b0be5-95fd-7776-9a98-70ab95462275 3
139+
__mo_index_secondary_019b0be5-95fd-77bc-a462-f9ff422590ec 1
140+
__mo_index_secondary_019b0be5-95fd-77b4-9add-05fb884cd701 1
141+
__mo_index_secondary_019b0be5-95fd-77ac-89bd-7e681c4689d8 1
142+
__mo_index_secondary_019b0be5-95fd-77a5-bc19-7b0dd32ae669 1
143+
__mo_index_secondary_019b0be5-95fd-7789-927c-8fbad1739d08 1
144+
DEALLOCATE PREPARE stmt;
145+
drop table t3;
146+
SET experimental_fulltext_index = 0;
147+
SET experimental_ivf_index = 0;
148+
SET experimental_hnsw_index = 0;
149+
drop database db;

‎test/distributed/cases/snapshot/clone/clone_in_alter_table_2.sql‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
SET experimental_fulltext_index = 1;
2+
SET experimental_ivf_index = 1;
3+
SET experimental_hnsw_index = 1;
4+
5+
drop database if exists db;
6+
create database db;
7+
use db;
8+
9+
-- case 1: primary table duplicate
10+
DROP TABLE IF EXISTS stress_alter_table;
11+
CREATE TABLE `stress_alter_table` (
12+
`md5_id` int NOT NULL,
13+
`b` varchar(65535) DEFAULT NULL,
14+
`delete_flag` int DEFAULT NULL,
15+
PRIMARY KEY (`md5_id`),
16+
FULLTEXT `fb`(`b`) WITH PARSER ngram,
17+
KEY `delete_flag_idx` (`delete_flag`)
18+
);
19+
20+
INSERT INTO stress_alter_table VALUES (1, 'this is a one', 0);
21+
22+
DELETE FROM `stress_alter_table` WHERE delete_flag = 0;
23+
INSERT INTO `stress_alter_table` (`md5_id`,`b`,`delete_flag`) VALUES (1,'this is a one',0);
24+
ALTER TABLE `stress_alter_table` MODIFY COLUMN delete_flag INT DEFAULT 0;
25+
-- @ignore:0
26+
select mo_ctl('dn','checkpoint','');
27+
SELECT COUNT(*) FROM `stress_alter_table`;
28+
UPDATE `stress_alter_table` SET delete_flag = 1 WHERE md5_id = '1';
29+
SELECT COUNT(*) FROM `stress_alter_table`;
30+
31+
drop table stress_alter_table;
32+
33+
-- case 2
34+
create table t1 (id bigint primary key, delete_flag int, a int, body varchar(10), title text, FULLTEXT (title, body), key(delete_flag));
35+
insert into t1 values(1,1,1, "body", "title");
36+
ALTER TABLE t1 MODIFY COLUMN a int DEFAULT NULL;
37+
38+
SET @inner_sql = (
39+
SELECT GROUP_CONCAT(
40+
DISTINCT CONCAT(
41+
'SELECT ''', mi.index_table_name,
42+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
43+
mi.index_table_name, '`'
44+
) SEPARATOR ' UNION ALL '
45+
)
46+
FROM mo_catalog.mo_indexes mi
47+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
48+
WHERE mt.relname = 't1'
49+
AND mt.reldatabase = 'db'
50+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
51+
AND mi.index_table_name IS NOT NULL
52+
AND mi.index_table_name != ''
53+
AND mi.column_name <> 'question'
54+
AND mi.column_name <> 'keyword'
55+
);
56+
57+
SET @sql = CONCAT(
58+
'SELECT * FROM (',
59+
@inner_sql,
60+
') AS t ORDER BY cnt DESC'
61+
);
62+
63+
PREPARE stmt FROM @sql;
64+
-- @ignore:0
65+
EXECUTE stmt;
66+
DEALLOCATE PREPARE stmt;
67+
68+
69+
drop table t1;
70+
71+
-- case 3
72+
73+
create table t2 (id int primary key, a int, b int, c int, key(a), key(b));
74+
insert into t2 values(1,1,1,1);
75+
ALTER TABLE t2 MODIFY COLUMN c int DEFAULT NULL;
76+
77+
SET @inner_sql = (
78+
SELECT GROUP_CONCAT(
79+
DISTINCT CONCAT(
80+
'SELECT ''', mi.index_table_name,
81+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
82+
mi.index_table_name, '`'
83+
) SEPARATOR ' UNION ALL '
84+
)
85+
FROM mo_catalog.mo_indexes mi
86+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
87+
WHERE mt.relname = 't2'
88+
AND mt.reldatabase = 'db'
89+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
90+
AND mi.index_table_name IS NOT NULL
91+
AND mi.index_table_name != ''
92+
AND mi.column_name <> 'question'
93+
AND mi.column_name <> 'keyword'
94+
);
95+
96+
SET @sql = CONCAT(
97+
'SELECT * FROM (',
98+
@inner_sql,
99+
') AS t ORDER BY cnt DESC'
100+
);
101+
102+
PREPARE stmt FROM @sql;
103+
-- @ignore:0
104+
EXECUTE stmt;
105+
DEALLOCATE PREPARE stmt;
106+
107+
drop table t2;
108+
109+
-- case 3
110+
111+
create table t3 (
112+
id bigint primary key,
113+
delete_flag int,
114+
a vecf32(3),
115+
b vecf32(3),
116+
body varchar(10),
117+
title text,
118+
FULLTEXT (title, body),
119+
key(delete_flag),
120+
key ivf using ivfflat (a),
121+
key idx using hnsw(b) op_type 'vector_l2_ops'
122+
);
123+
124+
insert into t3 values(1,1, '[0.1,0.2,0.3]','[0.1,0.2,0.3]', "body", "title");
125+
ALTER TABLE t3 MODIFY COLUMN delete_flag int DEFAULT NULL;
126+
127+
SET @inner_sql = (
128+
SELECT GROUP_CONCAT(
129+
DISTINCT CONCAT(
130+
'SELECT ''', mi.index_table_name,
131+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
132+
mi.index_table_name, '`'
133+
) SEPARATOR ' UNION ALL '
134+
)
135+
FROM mo_catalog.mo_indexes mi
136+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
137+
WHERE mt.relname = 't3'
138+
AND mt.reldatabase = 'db'
139+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
140+
AND mi.index_table_name IS NOT NULL
141+
AND mi.index_table_name != ''
142+
AND mi.column_name <> 'question'
143+
AND mi.column_name <> 'keyword'
144+
);
145+
146+
SET @sql = CONCAT(
147+
'SELECT * FROM (',
148+
@inner_sql,
149+
') AS t ORDER BY cnt DESC'
150+
);
151+
152+
PREPARE stmt FROM @sql;
153+
-- @ignore:0
154+
EXECUTE stmt;
155+
DEALLOCATE PREPARE stmt;
156+
157+
drop table t3;
158+
159+
SET experimental_fulltext_index = 0;
160+
SET experimental_ivf_index = 0;
161+
SET experimental_hnsw_index = 0;
162+
163+
drop database db;

0 commit comments

Comments
 (0)