Skip to content

Commit f4f63c7

Browse files
snjezasnjeza
committed
[performance] Import the large project
1 parent c204a8f commit f4f63c7

14 files changed

+234
-96
lines changed

‎bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelElement.java‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@
1313
*/
1414
package com.salesforce.bazel.eclipse.core.model;
1515

16+
import static java.lang.String.format;
1617
import static java.util.Objects.requireNonNull;
1718
import static java.util.Optional.ofNullable;
1819

1920
import java.util.Optional;
21+
import java.util.function.Supplier;
2022

2123
import org.eclipse.core.runtime.CoreException;
2224
import org.eclipse.core.runtime.IPath;
23-
import org.eclipse.core.runtime.OperationCanceledException;
24-
import org.eclipse.core.runtime.Status;
2525

2626
import com.salesforce.bazel.eclipse.core.model.cache.BazelElementInfoCache;
2727
import com.salesforce.bazel.sdk.model.BazelLabel;
@@ -129,24 +129,21 @@ protected final I getInfo() throws CoreException {
129129
if (info != null) {
130130
return info;
131131
}
132-
133132
// ensure the parent is loaded
134133
if (hasParent()) {
135134
getParent().getInfo();
136135
}
137-
138-
// loads can be potentially expensive; we synchronize on the location
139-
var location = getLocation();
140-
var openJob = new BazelElementOpenJob<>(location != null ? location : IPath.ROOT, this, infoCache);
141-
try {
142-
return openJob.open();
143-
} catch (InterruptedException e) {
144-
throw new OperationCanceledException("Interrupted while opening element.");
145-
} catch (CoreException e) {
146-
// wrap to provide better context
147-
throw new CoreException(
148-
Status.error(String.format("Error opening '%s': '%s'", location, e.getMessage()), e));
149-
}
136+
// store in cache
137+
Supplier<I> supplier = () -> {
138+
try {
139+
return requireNonNull(
140+
createInfo(),
141+
() -> format("invalid implementation of #createInfo in %s; must not return null!", getClass()));
142+
} catch (CoreException e) {
143+
throw new RuntimeException(e);
144+
}
145+
};
146+
return (I) infoCache.putOrGetCached(this, supplier);
150147
}
151148

152149
BazelElementInfoCache getInfoCache() {

‎bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelPackage.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@
99

1010
import java.nio.file.Path;
1111
import java.util.List;
12+
import java.util.Map;
1213
import java.util.Objects;
1314

1415
import org.eclipse.core.runtime.CoreException;
1516
import org.eclipse.core.runtime.IPath;
1617
import org.eclipse.core.runtime.Status;
1718

19+
import com.salesforce.bazel.sdk.command.querylight.Target;
1820
import com.salesforce.bazel.sdk.model.BazelLabel;
1921

2022
/**
@@ -71,6 +73,7 @@ public static boolean isBuildFileName(String fileName) {
7173
private final BazelWorkspace parent;
7274
private final BazelLabel label;
7375
private final IPath packagePath;
76+
private Map<String, Target> targets;
7477

7578
BazelPackage(BazelWorkspace parent, IPath packagePath) throws NullPointerException, IllegalArgumentException {
7679
this.packagePath =
@@ -86,8 +89,9 @@ protected BazelPackageInfo createInfo() throws CoreException {
8689
throw new CoreException(
8790
Status.error(format("Package '%s' does not exist in workspace '%s'!", label, parent.getName())));
8891
}
89-
90-
var targets = BazelPackageInfo.queryForTargets(this, getCommandExecutor());
92+
if (targets == null) {
93+
targets = BazelPackageInfo.queryForTargets(this, getCommandExecutor());
94+
}
9195
return new BazelPackageInfo(buildFile, this, targets);
9296
}
9397

@@ -344,4 +348,8 @@ public void rediscoverBazelProject() throws CoreException {
344348
getInfo();
345349
}
346350

351+
public void setTargets(Map<String, Target> targets) {
352+
this.targets = targets;
353+
}
354+
347355
}

‎bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelPackageInfo.java‎

Lines changed: 9 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
*/
1414
package com.salesforce.bazel.eclipse.core.model;
1515

16-
import static com.salesforce.bazel.eclipse.core.BazelCoreSharedContstants.BAZEL_NATURE_ID;
17-
import static com.salesforce.bazel.eclipse.core.model.BazelProject.hasOwnerPropertySetForLabel;
18-
import static com.salesforce.bazel.eclipse.core.model.BazelProject.hasWorkspaceRootPropertySetToLocation;
1916
import static java.lang.String.format;
2017
import static java.util.stream.Collectors.joining;
2118

@@ -55,20 +52,8 @@ public final class BazelPackageInfo extends BazelElementInfo {
5552
* @throws CoreException
5653
*/
5754
static IProject findProject(BazelPackage bazelPackage) throws CoreException {
58-
var workspaceRoot = bazelPackage.getBazelWorkspace().getLocation();
59-
// we don't care about the actual project name - we look for the property
60-
var projects = getEclipseWorkspaceRoot().getProjects();
61-
for (IProject project : projects) {
62-
if (project.isAccessible() // is open
63-
&& project.hasNature(BAZEL_NATURE_ID) // is a Bazel project
64-
&& hasWorkspaceRootPropertySetToLocation(project, workspaceRoot) // belongs to the workspace root
65-
&& hasOwnerPropertySetForLabel(project, bazelPackage.getLabel()) // represents the target
66-
) {
67-
return project;
68-
}
69-
}
70-
71-
return null;
55+
var bazelWorkspace = bazelPackage.getBazelWorkspace();
56+
return bazelWorkspace.getInfo().getProject(bazelPackage.getLabel());
7257
}
7358

7459
static Map<String, Target> queryForTargets(BazelPackage bazelPackage,
@@ -138,7 +123,7 @@ static Map<BazelPackage, Map<String, Target>> queryForTargets(BazelWorkspace baz
138123
private final BazelPackage bazelPackage;
139124
private final Map<String, Target> indexOfTargetInfoByTargetName;
140125

141-
private final BazelProject bazelProject;
126+
private BazelProject bazelProject;
142127

143128
private BazelVisibility defaultVisibility;
144129

@@ -147,22 +132,19 @@ static Map<BazelPackage, Map<String, Target>> queryForTargets(BazelWorkspace baz
147132
this.buildFile = buildFile;
148133
this.bazelPackage = bazelPackage;
149134
this.indexOfTargetInfoByTargetName = indexOfTargetInfoByTargetName;
150-
151-
// find project is expensive, do it only once when loading a package
152-
// (https://github.com/eclipseguru/bazel-eclipse/issues/8)
153-
var project = findProject(bazelPackage);
154-
if (project != null) {
155-
bazelProject = new BazelProject(project, bazelPackage.getModel());
156-
} else {
157-
bazelProject = null;
158-
}
159135
}
160136

161137
public BazelPackage getBazelPackage() {
162138
return bazelPackage;
163139
}
164140

165141
public BazelProject getBazelProject() throws CoreException {
142+
// find project is expensive, do it only once when loading a package
143+
// (https://github.com/eclipseguru/bazel-eclipse/issues/8)
144+
var project = findProject(bazelPackage);
145+
if (project != null) {
146+
bazelProject = new BazelProject(project, bazelPackage.getModel());
147+
}
166148
if (bazelProject != null) {
167149
return bazelProject;
168150
}

‎bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelTargetInfo.java‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
*/
1414
package com.salesforce.bazel.eclipse.core.model;
1515

16-
import static com.salesforce.bazel.eclipse.core.BazelCoreSharedContstants.BAZEL_NATURE_ID;
17-
import static com.salesforce.bazel.eclipse.core.model.BazelProject.hasOwnerPropertySetForLabel;
18-
import static com.salesforce.bazel.eclipse.core.model.BazelProject.hasWorkspaceRootPropertySetToLocation;
1916
import static java.lang.String.format;
2017
import static java.util.Objects.requireNonNull;
2118
import static java.util.stream.Collectors.toList;
@@ -46,26 +43,14 @@ public final class BazelTargetInfo extends BazelElementInfo {
4643
* @throws CoreException
4744
*/
4845
static IProject findProject(BazelTarget bazelTarget) throws CoreException {
49-
var workspaceRoot = bazelTarget.getBazelWorkspace().getLocation();
50-
// we don't care about the actual project name - we look for the property
51-
var projects = getEclipseWorkspaceRoot().getProjects();
52-
for (IProject project : projects) {
53-
if (project.isAccessible() // is open
54-
&& project.hasNature(BAZEL_NATURE_ID) // is a Bazel project
55-
&& hasWorkspaceRootPropertySetToLocation(project, workspaceRoot) // belongs to the workspace root
56-
&& hasOwnerPropertySetForLabel(project, bazelTarget.getLabel()) // represents the target
57-
) {
58-
return project;
59-
}
60-
}
61-
62-
return null;
46+
var bazelWorkspace = bazelTarget.getBazelWorkspace();
47+
return bazelWorkspace.getInfo().getProject(bazelTarget.getLabel());
6348
}
6449

6550
private final BazelTarget bazelTarget;
6651
private final String targetName;
6752
private final Target target;
68-
private final BazelProject bazelProject;
53+
private BazelProject bazelProject;
6954
private volatile BazelRuleAttributes ruleAttributes;
7055

7156
private List<IPath> ruleOutput;
@@ -86,18 +71,15 @@ public BazelTargetInfo(String targetName, BazelTarget bazelTarget, BazelPackageI
8671
getTargetName(),
8772
packageInfo.getBazelPackage().getLabel())));
8873
}
74+
}
8975

76+
public BazelProject getBazelProject() throws CoreException {
9077
// find project is expensive, do it only once when loading a package
9178
// (https://github.com/eclipseguru/bazel-eclipse/issues/8)
9279
var project = findProject(bazelTarget);
9380
if (project != null) {
9481
bazelProject = new BazelProject(project, bazelTarget.getModel());
95-
} else {
96-
bazelProject = null;
9782
}
98-
}
99-
100-
public BazelProject getBazelProject() throws CoreException {
10183
if (bazelProject != null) {
10284
return bazelProject;
10385
}

‎bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelWorkspace.java‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,25 @@
1717
import static com.salesforce.bazel.eclipse.core.BazelCoreSharedContstants.FILE_NAME_REPO_BAZEL;
1818
import static com.salesforce.bazel.eclipse.core.BazelCoreSharedContstants.FILE_NAME_WORKSPACE;
1919
import static com.salesforce.bazel.eclipse.core.BazelCoreSharedContstants.FILE_NAME_WORKSPACE_BAZEL;
20-
import static com.salesforce.bazel.eclipse.core.model.BazelPackageInfo.queryForTargets;
2120
import static java.lang.String.format;
2221
import static java.nio.file.Files.isDirectory;
2322
import static java.nio.file.Files.isRegularFile;
2423
import static java.util.Objects.requireNonNull;
2524
import static java.util.function.Predicate.not;
25+
import static java.util.stream.Collectors.joining;
2626
import static java.util.stream.Collectors.toList;
2727

2828
import java.nio.file.Path;
2929
import java.util.Collection;
3030
import java.util.Collections;
31+
import java.util.HashMap;
3132
import java.util.List;
33+
import java.util.Map;
3234
import java.util.Objects;
3335
import java.util.function.Predicate;
3436
import java.util.stream.Stream;
3537

38+
import org.eclipse.core.resources.IProject;
3639
import org.eclipse.core.runtime.CoreException;
3740
import org.eclipse.core.runtime.IPath;
3841
import org.eclipse.core.runtime.Status;
@@ -43,6 +46,8 @@
4346
import com.salesforce.bazel.eclipse.core.projectview.BazelProjectView;
4447
import com.salesforce.bazel.sdk.BazelVersion;
4548
import com.salesforce.bazel.sdk.command.BazelBinary;
49+
import com.salesforce.bazel.sdk.command.BazelQueryForTargetProtoCommand;
50+
import com.salesforce.bazel.sdk.command.querylight.Target;
4651
import com.salesforce.bazel.sdk.model.BazelLabel;
4752

4853
/**
@@ -218,6 +223,11 @@ public boolean exists() {
218223
return isDirectory(path) && (findWorkspaceFile(path) != null);
219224
}
220225

226+
public IProject findProject(BazelPackage bazelPackage) throws CoreException {
227+
var info = bazelPackage.getBazelWorkspace().getInfo();
228+
return info.getProject(bazelPackage.getLabel());
229+
}
230+
221231
/**
222232
* @return a collection of all targets loaded in memory
223233
*/
@@ -616,7 +626,7 @@ public void open(Collection<BazelPackage> bazelPackages) throws CoreException {
616626
}
617627

618628
// open all closed projects
619-
var targetsByPackage = queryForTargets(this, closedPackages, getCommandExecutor());
629+
var targetsByPackage = queryForTargetsWithDependencies(this, closedPackages, getCommandExecutor());
620630
for (BazelPackage bazelPackage : closedPackages) {
621631
if (bazelPackage.hasInfo()) {
622632
continue;
@@ -634,6 +644,11 @@ public void open(Collection<BazelPackage> bazelPackages) throws CoreException {
634644
LOG.warn("Package '{}' does not have a BUILD file, skipping", bazelPackage.getLabel());
635645
continue;
636646
}
647+
bazelPackage.setTargets(targets);
648+
if (!bazelPackage.hasInfo()) {
649+
// getting the info loads the package avoiding unnecessary double loads
650+
bazelPackage.getInfo();
651+
}
637652

638653
// Create PackageInfo directly with the batched query results
639654
var packageInfo = new BazelPackageInfo(buildFile, bazelPackage, targets);
@@ -644,7 +659,77 @@ public void open(Collection<BazelPackage> bazelPackages) throws CoreException {
644659
}
645660
}
646661

662+
public Map<BazelPackage, Map<String, Target>> queryForTargetsWithDependencies(BazelWorkspace bazelWorkspace,
663+
Collection<BazelPackage> bazelPackages, BazelElementCommandExecutor bazelElementCommandExecutor)
664+
throws CoreException {
665+
// bazel query 'kind(rule, deps(//foo:all + //bar:all))"'
666+
667+
if (bazelPackages.isEmpty()) {
668+
return Collections.emptyMap();
669+
}
670+
var workspaceRoot = bazelWorkspace.getLocation().toPath();
671+
var query = bazelPackages.stream()
672+
.map(bazelPackage -> format("//%s:all", bazelPackage.getWorkspaceRelativePath()))
673+
.collect(joining(" + "));
674+
675+
Map<String, BazelPackage> bazelPackageByWorkspaceRelativePath = new HashMap<>();
676+
bazelPackages.stream()
677+
.forEach(p -> bazelPackageByWorkspaceRelativePath.put(p.getWorkspaceRelativePath().toString(), p));
678+
679+
query = "kind(rule, deps(" + query + "))";
680+
Map<BazelPackage, Map<String, Target>> result = new HashMap<>();
681+
LOG.debug("{}: querying Bazel for list of targets from: {}", bazelWorkspace, query);
682+
var queryResult = bazelElementCommandExecutor.runQueryWithoutLock(
683+
new BazelQueryForTargetProtoCommand(
684+
workspaceRoot,
685+
query,
686+
true /* keep going */,
687+
List.of("--noproto:locations", "--noproto:default_values", "--noimplicit_deps", "--notool_deps"),
688+
format(
689+
"Loading targets for %d %s",
690+
bazelPackages.size(),
691+
bazelPackages.size() == 1 ? "package" : "packages")));
692+
for (Target target : queryResult) {
693+
if (!target.hasRule()) {
694+
LOG.trace("{}: ignoring target: {}", bazelWorkspace, target);
695+
System.out.println();
696+
continue;
697+
}
698+
699+
try {
700+
BazelLabel.validateLabelPath(target.rule().name(), true);
701+
} catch (Exception e) {
702+
LOG.trace("{}: ignoring target: {}", bazelWorkspace, target);
703+
continue;
704+
}
705+
LOG.trace("{}: found target: {}", bazelWorkspace, target);
706+
var targetLabel = new BazelLabel(target.rule().name());
707+
708+
var bazelPackage = bazelPackageByWorkspaceRelativePath.get(targetLabel.getPackagePath());
709+
if (bazelPackage == null) {
710+
// LOG.debug("{}: ignoring target for unknown package: {}", bazelWorkspace, targetLabel);
711+
// continue;
712+
var packageLabel = targetLabel.getPackageLabel();
713+
if (bazelWorkspace.isRootedAtThisWorkspace(packageLabel)) {
714+
bazelPackage = bazelWorkspace.getBazelPackage(packageLabel);
715+
}
716+
}
717+
if (bazelPackage == null) {
718+
LOG.debug("{}: ignoring target for unknown package: {}", bazelWorkspace, targetLabel);
719+
continue;
720+
}
721+
if (!result.containsKey(bazelPackage)) {
722+
result.put(bazelPackage, new HashMap<>());
723+
}
724+
725+
var targetName = targetLabel.getTargetName();
726+
result.get(bazelPackage).put(targetName, target);
727+
}
728+
return result;
729+
}
730+
647731
Path workspacePath() {
648732
return root.toPath();
649733
}
734+
650735
}

0 commit comments

Comments
 (0)