Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ public class LocalReplica extends Replica {
@SerializedName(value = "rds", alternate = {"remoteDataSize"})
private volatile long remoteDataSize = 0;
@SerializedName(value = "ris", alternate = {"remoteInvertedIndexSize"})
private Long remoteInvertedIndexSize = 0L;
private long remoteInvertedIndexSize = 0L;
@SerializedName(value = "rss", alternate = {"remoteSegmentSize"})
private Long remoteSegmentSize = 0L;
private long remoteSegmentSize = 0L;

// the last load failed version
@SerializedName(value = "lfv", alternate = {"lastFailedVersion"})
Expand Down Expand Up @@ -207,7 +207,7 @@ public void setRemoteDataSize(long remoteDataSize) {
}

@Override
public Long getRemoteInvertedIndexSize() {
public long getRemoteInvertedIndexSize() {
return remoteInvertedIndexSize;
}

Expand All @@ -217,7 +217,7 @@ public void setRemoteInvertedIndexSize(long remoteInvertedIndexSize) {
}

@Override
public Long getRemoteSegmentSize() {
public long getRemoteSegmentSize() {
return remoteSegmentSize;
}

Expand Down
13 changes: 13 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTablet.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
public class LocalTablet extends Tablet {
private static final Logger LOG = LogManager.getLogger(LocalTablet.class);

@SerializedName(value = "lastCheckTime")
private long lastCheckTime;

// cooldown conf
@SerializedName(value = "cri", alternate = {"cooldownReplicaId"})
private long cooldownReplicaId = -1;
Expand Down Expand Up @@ -148,4 +151,14 @@ protected long getLastTimeNoPathForNewReplica() {
public void setLastTimeNoPathForNewReplica(long lastTimeNoPathForNewReplica) {
this.lastTimeNoPathForNewReplica = lastTimeNoPathForNewReplica;
}

@Override
public long getLastCheckTime() {
return lastCheckTime;
}

@Override
public void setLastCheckTime(long lastCheckTime) {
this.lastCheckTime = lastCheckTime;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,11 @@ public static class ReplicaContext {
@Setter
@Getter
@SerializedName(value = "lis", alternate = {"localInvertedIndexSize"})
private Long localInvertedIndexSize = 0L;
private long localInvertedIndexSize = 0L;
@Setter
@Getter
@SerializedName(value = "lss", alternate = {"localSegmentSize"})
private Long localSegmentSize = 0L;
private long localSegmentSize = 0L;

public Replica() {
}
Expand Down Expand Up @@ -199,7 +199,7 @@ public void setRemoteDataSize(long remoteDataSize) {
}
}

public Long getRemoteInvertedIndexSize() {
public long getRemoteInvertedIndexSize() {
return 0L;
}

Expand All @@ -209,7 +209,7 @@ public void setRemoteInvertedIndexSize(long remoteInvertedIndexSize) {
}
}

public Long getRemoteSegmentSize() {
public long getRemoteSegmentSize() {
return 0L;
}

Expand Down
10 changes: 9 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
/**
* This class represents the olap tablet related metadata.
*/
public class Tablet extends MetaObject {
public class Tablet {
private static final Logger LOG = LogManager.getLogger(Tablet.class);
// if current version count of replica is mor than
// QUERYABLE_TIMES_OF_MIN_VERSION_COUNT times the minimum version count,
Expand Down Expand Up @@ -889,4 +889,12 @@ public void setLastTimeNoPathForNewReplica(long lastTimeNoPathForNewReplica) {
throw new UnsupportedOperationException("setLastTimeNoPathForNewReplica is not supported in Tablet");
}
}

public long getLastCheckTime() {
return -1;
}

public void setLastCheckTime(long lastCheckTime) {
throw new UnsupportedOperationException("setLastCheckTime is not supported in Tablet");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public class CloudReplica extends Replica {

private static final Random rand = new Random();

private Map<String, List<Long>> memClusterToBackends = new ConcurrentHashMap<String, List<Long>>();
private Map<String, List<Long>> memClusterToBackends = null;

// clusterId, secondaryBe, changeTimestamp
private Map<String, Pair<Long, Long>> secondaryClusterToBackends
Expand Down Expand Up @@ -311,6 +311,7 @@ private long getBackendIdImpl(String clusterId) throws ComputeGroupException {
int indexRand = rand.nextInt(Config.cloud_replica_num);
int coldReadRand = rand.nextInt(100);
boolean allowColdRead = coldReadRand < Config.cloud_cold_read_percent;
initMemClusterToBackends();
boolean replicaEnough = memClusterToBackends.get(clusterId) != null
&& memClusterToBackends.get(clusterId).size() > indexRand;

Expand Down Expand Up @@ -470,7 +471,18 @@ private long getIndexByBeNum(long hashValue, int beNum) {
return (hashValue % beNum + beNum) % beNum;
}

public List<Long> hashReplicaToBes(String clusterId, boolean isBackGround, int replicaNum)
private void initMemClusterToBackends() {
// the enable_cloud_multi_replica is not used now
if (memClusterToBackends == null) {
synchronized (this) {
if (memClusterToBackends == null) {
memClusterToBackends = new ConcurrentHashMap<>();
}
}
}
}

private List<Long> hashReplicaToBes(String clusterId, boolean isBackGround, int replicaNum)
throws ComputeGroupException {
// TODO(luwei) list should be sorted
List<Backend> clusterBes = ((CloudSystemInfoService) Env.getCurrentSystemInfo())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ public class ConsistencyChecker extends MasterDaemon {

private static final Comparator<MetaObject> COMPARATOR =
(first, second) -> Long.signum(first.getLastCheckTime() - second.getLastCheckTime());
private static final Comparator<Tablet> TABLET_COMPARATOR =
(first, second) -> Long.signum(first.getLastCheckTime() - second.getLastCheckTime());

// tabletId -> job
private Map<Long, CheckConsistencyJob> jobs;
Expand Down Expand Up @@ -315,12 +317,12 @@ private List<Long> chooseTablets() {
MaterializedIndex index = (MaterializedIndex) chosenOne;

// sort tablets
Queue<MetaObject> tabletQueue
= new PriorityQueue<>(Math.max(index.getTablets().size(), 1), COMPARATOR);
Queue<Tablet> tabletQueue = new PriorityQueue<>(Math.max(index.getTablets().size(), 1),
TABLET_COMPARATOR);
tabletQueue.addAll(index.getTablets());
Tablet tablet = null;

while ((chosenOne = tabletQueue.poll()) != null) {
Tablet tablet = (Tablet) chosenOne;
while ((tablet = tabletQueue.poll()) != null) {
long chosenTabletId = tablet.getId();

if (this.jobs.containsKey(chosenTabletId)) {
Expand Down
Loading