Skip to content

Commit 6e0a6a0

Browse files
mymeiyimymeiyi
authored
[opt](fe) modify TabletInvertedIndex to local and cloud to reduce memory (#59683)
1 parent a28e495 commit 6e0a6a0

File tree

17 files changed

+1238
-995
lines changed

17 files changed

+1238
-995
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -740,7 +740,7 @@ public Env(boolean isCheckpointCatalog) {
740740
this.feSessionMgr = new FESessionMgr();
741741
this.temporaryTableMgr = new TemporaryTableMgr();
742742
this.aliveSessionSet = Sets.newConcurrentHashSet();
743-
this.tabletInvertedIndex = new TabletInvertedIndex();
743+
this.tabletInvertedIndex = EnvFactory.getInstance().createTabletInvertedIndex();
744744
this.colocateTableIndex = new ColocateTableIndex();
745745
this.recycleBin = new CatalogRecycleBin();
746746
this.functionSet = new FunctionSet();

‎fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ public SystemInfoService createSystemInfoService() {
8686
return new SystemInfoService();
8787
}
8888

89+
public TabletInvertedIndex createTabletInvertedIndex() {
90+
return new LocalTabletInvertedIndex();
91+
}
92+
8993
public Type getPartitionClass() {
9094
return Partition.class;
9195
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java‎

Lines changed: 1046 additions & 0 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java‎

Lines changed: 31 additions & 972 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.doris.catalog.Replica;
2929
import org.apache.doris.catalog.ReplicaAllocation;
3030
import org.apache.doris.catalog.Tablet;
31+
import org.apache.doris.catalog.TabletInvertedIndex;
3132
import org.apache.doris.cloud.common.util.CloudPropertyAnalyzer;
3233
import org.apache.doris.cloud.datasource.CloudInternalCatalog;
3334
import org.apache.doris.cloud.load.CleanCopyJobScheduler;
@@ -83,6 +84,11 @@ public SystemInfoService createSystemInfoService() {
8384
return new CloudSystemInfoService();
8485
}
8586

87+
@Override
88+
public TabletInvertedIndex createTabletInvertedIndex() {
89+
return new CloudTabletInvertedIndex();
90+
}
91+
8692
@Override
8793
public Type getPartitionClass() {
8894
return CloudPartition.class;

‎fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.cloud.catalog;
19+
20+
import org.apache.doris.catalog.Replica;
21+
import org.apache.doris.catalog.TabletInvertedIndex;
22+
23+
import com.google.common.base.Preconditions;
24+
import com.google.common.collect.Lists;
25+
import com.google.common.collect.Maps;
26+
import org.apache.logging.log4j.LogManager;
27+
import org.apache.logging.log4j.Logger;
28+
29+
import java.util.List;
30+
import java.util.Map;
31+
32+
public class CloudTabletInvertedIndex extends TabletInvertedIndex {
33+
private static final Logger LOG = LogManager.getLogger(CloudTabletInvertedIndex.class);
34+
35+
// tablet id -> replica
36+
// for cloud mode, no need to know the replica's backend
37+
private Map<Long, Replica> replicaMetaMap = Maps.newHashMap();
38+
39+
public CloudTabletInvertedIndex() {
40+
super();
41+
}
42+
43+
@Override
44+
public List<Replica> getReplicas(Long tabletId) {
45+
long stamp = readLock();
46+
try {
47+
if (replicaMetaMap.containsKey(tabletId)) {
48+
return Lists.newArrayList(replicaMetaMap.get(tabletId));
49+
}
50+
return Lists.newArrayList();
51+
} finally {
52+
readUnlock(stamp);
53+
}
54+
}
55+
56+
@Override
57+
public void deleteTablet(long tabletId) {
58+
long stamp = writeLock();
59+
try {
60+
replicaMetaMap.remove(tabletId);
61+
tabletMetaMap.remove(tabletId);
62+
if (LOG.isDebugEnabled()) {
63+
LOG.debug("delete tablet: {}", tabletId);
64+
}
65+
} finally {
66+
writeUnlock(stamp);
67+
}
68+
}
69+
70+
@Override
71+
public void addReplica(long tabletId, Replica replica) {
72+
long stamp = writeLock();
73+
try {
74+
Preconditions.checkState(tabletMetaMap.containsKey(tabletId),
75+
"tablet " + tabletId + " not exists, replica " + replica.getId());
76+
replicaMetaMap.put(tabletId, replica);
77+
if (LOG.isDebugEnabled()) {
78+
LOG.debug("add replica {} of tablet {}", replica.getId(), tabletId);
79+
}
80+
} finally {
81+
writeUnlock(stamp);
82+
}
83+
}
84+
85+
@Override
86+
public void deleteReplica(long tabletId, long backendId) {
87+
long stamp = writeLock();
88+
try {
89+
Preconditions.checkState(tabletMetaMap.containsKey(tabletId), "tablet " + tabletId + " not exists");
90+
if (replicaMetaMap.containsKey(tabletId)) {
91+
Replica replica = replicaMetaMap.remove(tabletId);
92+
if (LOG.isDebugEnabled()) {
93+
LOG.debug("delete replica {} of tablet {}", replica.getId(), tabletId);
94+
}
95+
} else {
96+
// this may happen when fe restart after tablet is empty(bug cause)
97+
// add log instead of assertion to observe
98+
LOG.error("tablet[{}] contains no replica in inverted index", tabletId);
99+
}
100+
} finally {
101+
writeUnlock(stamp);
102+
}
103+
}
104+
105+
@Override
106+
public Replica getReplica(long tabletId, long backendId) {
107+
long stamp = readLock();
108+
try {
109+
Preconditions.checkState(tabletMetaMap.containsKey(tabletId), "tablet " + tabletId + " not exists");
110+
return replicaMetaMap.get(tabletId);
111+
} finally {
112+
readUnlock(stamp);
113+
}
114+
}
115+
116+
@Override
117+
public List<Replica> getReplicasByTabletId(long tabletId) {
118+
long stamp = readLock();
119+
try {
120+
if (replicaMetaMap.containsKey(tabletId)) {
121+
return Lists.newArrayList(replicaMetaMap.get(tabletId));
122+
}
123+
return Lists.newArrayList();
124+
} finally {
125+
readUnlock(stamp);
126+
}
127+
}
128+
129+
@Override
130+
protected void innerClear() {
131+
replicaMetaMap.clear();
132+
}
133+
}

‎fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.catalog.BrokerMgr;
2222
import org.apache.doris.catalog.Database;
2323
import org.apache.doris.catalog.Env;
24+
import org.apache.doris.catalog.LocalTabletInvertedIndex;
2425
import org.apache.doris.catalog.MaterializedIndex;
2526
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
2627
import org.apache.doris.catalog.OlapTable;
@@ -92,7 +93,7 @@ public class BackupHandlerTest {
9293

9394
private String tmpPath = "./tmp" + System.currentTimeMillis();
9495

95-
private TabletInvertedIndex invertedIndex = new TabletInvertedIndex();
96+
private TabletInvertedIndex invertedIndex = new LocalTabletInvertedIndex();
9697

9798
@Before
9899
public void setUp() throws Exception {

‎fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.junit.Assert;
3333
import org.junit.jupiter.api.Test;
3434

35-
import java.util.HashSet;
3635
import java.util.Map;
3736
import java.util.Set;
3837
import java.util.UUID;
@@ -60,8 +59,6 @@ public void testDuplicateCreateTable() throws Exception {
6059
+ "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1','colocate_with'='test'); ";
6160
createTable(sql);
6261
Set<Long> tabletIdSetAfterCreateFirstTable = env.getTabletInvertedIndex().getReplicaMetaTable().rowKeySet();
63-
Set<TabletMeta> tabletMetaSetBeforeCreateFirstTable =
64-
new HashSet<>(env.getTabletInvertedIndex().getTabletMetaTable().values());
6562
Set<Long> colocateTableIdBeforeCreateFirstTable = env.getColocateTableIndex().getTable2Group().keySet();
6663
Assert.assertTrue(colocateTableIdBeforeCreateFirstTable.size() > 0);
6764
Assert.assertTrue(tabletIdSetAfterCreateFirstTable.size() > 0);
@@ -71,13 +68,10 @@ public void testDuplicateCreateTable() throws Exception {
7168
Set<Long> tabletIdSetAfterDuplicateCreateTable1 = env.getTabletInvertedIndex().getReplicaMetaTable().rowKeySet();
7269
Set<Long> tabletIdSetAfterDuplicateCreateTable2 = env.getTabletInvertedIndex().getBackingReplicaMetaTable().columnKeySet();
7370
Set<Long> tabletIdSetAfterDuplicateCreateTable3 = env.getTabletInvertedIndex().getTabletMetaMap().keySet();
74-
Set<TabletMeta> tabletIdSetAfterDuplicateCreateTable4 =
75-
new HashSet<>(env.getTabletInvertedIndex().getTabletMetaTable().values());
7671

7772
Assert.assertEquals(tabletIdSetAfterCreateFirstTable, tabletIdSetAfterDuplicateCreateTable1);
7873
Assert.assertEquals(tabletIdSetAfterCreateFirstTable, tabletIdSetAfterDuplicateCreateTable2);
7974
Assert.assertEquals(tabletIdSetAfterCreateFirstTable, tabletIdSetAfterDuplicateCreateTable3);
80-
Assert.assertEquals(tabletMetaSetBeforeCreateFirstTable, tabletIdSetAfterDuplicateCreateTable4);
8175

8276
// check whether table id is cleared from colocate group after duplicate create table
8377
Set<Long> colocateTableIdAfterCreateFirstTable = env.getColocateTableIndex().getTable2Group().keySet();

‎fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public class TabletTest {
5353

5454
@Before
5555
public void makeTablet() {
56-
invertedIndex = new TabletInvertedIndex();
56+
invertedIndex = new LocalTabletInvertedIndex();
5757
infoService = new SystemInfoService();
5858
for (long beId = 1L; beId <= 4L; beId++) {
5959
Backend be = new Backend(beId, "127.0.0." + beId, 8030);

‎fe/fe-core/src/test/java/org/apache/doris/clone/ClusterLoadStatisticsTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.catalog.DiskInfo;
2121
import org.apache.doris.catalog.Env;
2222
import org.apache.doris.catalog.LocalReplica;
23+
import org.apache.doris.catalog.LocalTabletInvertedIndex;
2324
import org.apache.doris.catalog.Replica.ReplicaState;
2425
import org.apache.doris.catalog.TabletInvertedIndex;
2526
import org.apache.doris.catalog.TabletMeta;
@@ -148,7 +149,7 @@ public void setUp() {
148149
systemInfoService.addBackend(be4);
149150

150151
// tablet
151-
invertedIndex = new TabletInvertedIndex();
152+
invertedIndex = new LocalTabletInvertedIndex();
152153

153154
invertedIndex.addTablet(50000, new TabletMeta(1, 2, 3, 4, 5, TStorageMedium.HDD));
154155
invertedIndex.addReplica(50000, new LocalReplica(50001, be1.getId(), 0, ReplicaState.NORMAL));

0 commit comments

Comments
 (0)