Skip to content

Commit 62bfb70

Browse files
michaelnebelmichaelnebel
committed
C#: Refactor some of the feed logic into a separate class.
1 parent 2f24162 commit 62bfb70

3 files changed

Lines changed: 156 additions & 117 deletions

File tree

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Collections.Immutable;
4+
using System.Linq;
5+
using System.Text;
6+
using System.Text.RegularExpressions;
7+
using Semmle.Util;
8+
using Semmle.Util.Logging;
9+
10+
namespace Semmle.Extraction.CSharp.DependencyFetching
11+
{
12+
internal sealed partial class FeedManager : IDisposable
13+
{
14+
private readonly ILogger logger;
15+
private readonly IDotNet dotnet;
16+
private readonly DependencyDirectory emptyPackageDirectory;
17+
18+
public ImmutableHashSet<string> PrivateRegistryFeeds { get; }
19+
public bool HasPrivateRegistryFeeds { get; }
20+
public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
21+
22+
public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy)
23+
{
24+
this.logger = logger;
25+
this.dotnet = dotnet;
26+
PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
27+
HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0;
28+
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
29+
}
30+
31+
private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
32+
{
33+
var results = getNugetFeeds();
34+
var regex = EnabledNugetFeed();
35+
foreach (var result in results)
36+
{
37+
var match = regex.Match(result);
38+
if (!match.Success)
39+
{
40+
logger.LogError($"Failed to parse feed from '{result}'");
41+
continue;
42+
}
43+
44+
var url = match.Groups[1].Value;
45+
if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
46+
!url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
47+
{
48+
logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
49+
continue;
50+
}
51+
52+
if (!string.IsNullOrWhiteSpace(url))
53+
{
54+
yield return url;
55+
}
56+
}
57+
}
58+
59+
public IEnumerable<string> GetFeedsFromFolder(string folderPath) =>
60+
GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folderPath));
61+
62+
63+
public IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
64+
GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath));
65+
66+
private string FeedsToRestoreArgument(IEnumerable<string> feeds)
67+
{
68+
// If there are no feeds, we want to override any default feeds that `dotnet restore` would use by passing a dummy source argument.
69+
if (!feeds.Any())
70+
{
71+
return $" -s \"{emptyPackageDirectory.DirInfo.FullName}\"";
72+
}
73+
74+
// Add package sources. If any are present, they override all sources specified in
75+
// the configuration file(s).
76+
var feedArgs = new StringBuilder();
77+
foreach (var feed in feeds)
78+
{
79+
feedArgs.Append($" -s \"{feed}\"");
80+
}
81+
82+
return feedArgs.ToString();
83+
}
84+
85+
/// <summary>
86+
/// Constructs the list of NuGet sources to use for this restore.
87+
/// (1) Use the feeds we get from `dotnet nuget list source`
88+
/// (2) Use private registries, if they are configured
89+
/// </summary>
90+
/// <param name="path">Path to project/solution</param>
91+
/// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
92+
/// <returns>A string representing the NuGet sources argument for the restore command.</returns>
93+
public string? MakeRestoreSourcesArgument(string path, HashSet<string> reachableFeeds)
94+
{
95+
// Do not construct an set of explicit NuGet sources to use for restore.
96+
if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
97+
{
98+
return null;
99+
}
100+
101+
// Find the path specific feeds.
102+
var folder = FileUtils.GetDirectoryName(path, logger);
103+
var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet<string>();
104+
105+
if (HasPrivateRegistryFeeds)
106+
{
107+
feedsToConsider.UnionWith(PrivateRegistryFeeds);
108+
}
109+
110+
var feedsToUse = CheckNugetFeedResponsiveness
111+
? feedsToConsider.Where(reachableFeeds.Contains)
112+
: feedsToConsider;
113+
114+
return FeedsToRestoreArgument(feedsToUse);
115+
}
116+
117+
[GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
118+
private static partial Regex EnabledNugetFeed();
119+
120+
public void Dispose()
121+
{
122+
emptyPackageDirectory.Dispose();
123+
}
124+
}
125+
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs‎

Lines changed: 18 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,13 @@ internal sealed partial class NugetPackageRestorer : IDisposable
2828
private readonly IDiagnosticsWriter diagnosticsWriter;
2929
private readonly DependencyDirectory legacyPackageDirectory;
3030
private readonly DependencyDirectory missingPackageDirectory;
31-
private readonly DependencyDirectory emptyPackageDirectory;
3231
private readonly ILogger logger;
3332
private readonly ICompilationInfoContainer compilationInfoContainer;
34-
private readonly bool checkNugetFeedResponsiveness = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
35-
private readonly ImmutableHashSet<string> privateRegistryFeeds;
36-
private readonly bool hasPrivateRegistryFeeds;
33+
private readonly FeedManager feedManager;
3734

3835
public DependencyDirectory PackageDirectory { get; }
3936

37+
4038
public NugetPackageRestorer(
4139
FileProvider fileProvider,
4240
FileContent fileContent,
@@ -50,16 +48,14 @@ public NugetPackageRestorer(
5048
this.fileContent = fileContent;
5149
this.dotnet = dotnet;
5250
this.dependabotProxy = dependabotProxy;
53-
this.privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
54-
this.hasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
5551
this.diagnosticsWriter = diagnosticsWriter;
5652
this.logger = logger;
5753
this.compilationInfoContainer = compilationInfoContainer;
5854

5955
PackageDirectory = new DependencyDirectory("packages", "package", logger);
6056
legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger);
6157
missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger);
62-
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
58+
feedManager = new FeedManager(logger, dotnet, dependabotProxy);
6359
}
6460

6561
public string? TryRestore(string package)
@@ -118,8 +114,8 @@ public DirectoryInfo[] GetOrderedPackageVersionSubDirectories(string packagePath
118114
public HashSet<AssemblyLookupLocation> Restore()
119115
{
120116
var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
121-
logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
122-
compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
117+
logger.LogInfo($"Checking NuGet feed responsiveness: {feedManager.CheckNugetFeedResponsiveness}");
118+
compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", feedManager.CheckNugetFeedResponsiveness ? "1" : "0"));
123119

124120
HashSet<string> explicitFeeds = [];
125121
HashSet<string> reachableFeeds = [];
@@ -131,7 +127,7 @@ public HashSet<AssemblyLookupLocation> Restore()
131127
// (including inherited ones) from other locations on the host outside of the working directory.
132128
(explicitFeeds, var allFeeds) = GetAllFeeds();
133129

134-
if (checkNugetFeedResponsiveness)
130+
if (feedManager.CheckNugetFeedResponsiveness)
135131
{
136132
var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet();
137133

@@ -215,7 +211,7 @@ public HashSet<AssemblyLookupLocation> Restore()
215211

216212
var usedPackageNames = GetAllUsedPackageDirNames(dependencies);
217213

218-
var missingPackageLocation = checkNugetFeedResponsiveness
214+
var missingPackageLocation = feedManager.CheckNugetFeedResponsiveness
219215
? DownloadMissingPackagesFromSpecificFeeds(usedPackageNames, explicitFeeds)
220216
: DownloadMissingPackages(usedPackageNames);
221217

@@ -264,7 +260,7 @@ private List<string> GetReachableNuGetFeeds(HashSet<string> feedsToCheck, bool i
264260

265261
private bool IsDefaultFeedReachable()
266262
{
267-
if (checkNugetFeedResponsiveness)
263+
if (feedManager.CheckNugetFeedResponsiveness)
268264
{
269265
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
270266
return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount, out var _);
@@ -321,7 +317,7 @@ private IEnumerable<string> RestoreSolutions(HashSet<string> reachableFeeds, out
321317
var projects = fileProvider.Solutions.SelectMany(solution =>
322318
{
323319
logger.LogInfo($"Restoring solution {solution}...");
324-
var nugetSources = MakeRestoreSourcesArgument(solution, reachableFeeds);
320+
var nugetSources = feedManager.MakeRestoreSourcesArgument(solution, reachableFeeds);
325321
var res = dotnet.Restore(new(solution, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows));
326322
if (res.Success)
327323
{
@@ -346,57 +342,6 @@ private IEnumerable<string> RestoreSolutions(HashSet<string> reachableFeeds, out
346342
return projects;
347343
}
348344

349-
private string FeedsToRestoreArgument(IEnumerable<string> feeds)
350-
{
351-
// If there are no feeds, we want to override any default feeds that `dotnet restore` would use by passing a dummy source argument.
352-
if (!feeds.Any())
353-
{
354-
return $" -s \"{emptyPackageDirectory.DirInfo.FullName}\"";
355-
}
356-
357-
// Add package sources. If any are present, they override all sources specified in
358-
// the configuration file(s).
359-
var feedArgs = new StringBuilder();
360-
foreach (var feed in feeds)
361-
{
362-
feedArgs.Append($" -s \"{feed}\"");
363-
}
364-
365-
return feedArgs.ToString();
366-
}
367-
368-
/// <summary>
369-
/// Constructs the list of NuGet sources to use for this restore.
370-
/// (1) Use the feeds we get from `dotnet nuget list source`
371-
/// (2) Use private registries, if they are configured
372-
/// </summary>
373-
/// <param name="path">Path to project/solution</param>
374-
/// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
375-
/// <returns>A string representing the NuGet sources argument for the restore command.</returns>
376-
private string? MakeRestoreSourcesArgument(string path, HashSet<string> reachableFeeds)
377-
{
378-
// Do not construct an set of explicit NuGet sources to use for restore.
379-
if (!checkNugetFeedResponsiveness && !hasPrivateRegistryFeeds)
380-
{
381-
return null;
382-
}
383-
384-
// Find the path specific feeds.
385-
var folder = GetDirectoryName(path);
386-
var feedsToConsider = folder is not null ? GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder)).ToHashSet() : [];
387-
388-
if (hasPrivateRegistryFeeds)
389-
{
390-
feedsToConsider.UnionWith(privateRegistryFeeds);
391-
}
392-
393-
var feedsToUse = checkNugetFeedResponsiveness
394-
? feedsToConsider.Where(reachableFeeds.Contains)
395-
: feedsToConsider;
396-
397-
return FeedsToRestoreArgument(feedsToUse);
398-
}
399-
400345
/// <summary>
401346
/// Executes `dotnet restore` on all projects in projects.
402347
/// This is done in parallel for performance reasons.
@@ -421,7 +366,7 @@ private void RestoreProjects(IEnumerable<string> projects, HashSet<string> reach
421366
foreach (var project in projectGroup)
422367
{
423368
logger.LogInfo($"Restoring project {project}...");
424-
var nugetSources = MakeRestoreSourcesArgument(project, reachableFeeds);
369+
var nugetSources = feedManager.MakeRestoreSourcesArgument(project, reachableFeeds);
425370
var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows));
426371
assets.AddDependenciesRange(res.AssetsFilePaths);
427372
lock (sync)
@@ -899,47 +844,6 @@ private void EmitUnreachableFeedsDiagnostics(bool allFeedsReachable)
899844
compilationInfoContainer.CompilationInfos.Add(("All NuGet feeds reachable", allFeedsReachable ? "1" : "0"));
900845
}
901846

902-
private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
903-
{
904-
var results = getNugetFeeds();
905-
var regex = EnabledNugetFeed();
906-
foreach (var result in results)
907-
{
908-
var match = regex.Match(result);
909-
if (!match.Success)
910-
{
911-
logger.LogError($"Failed to parse feed from '{result}'");
912-
continue;
913-
}
914-
915-
var url = match.Groups[1].Value;
916-
if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
917-
!url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
918-
{
919-
logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
920-
continue;
921-
}
922-
923-
if (!string.IsNullOrWhiteSpace(url))
924-
{
925-
yield return url;
926-
}
927-
}
928-
}
929-
930-
private string? GetDirectoryName(string path)
931-
{
932-
try
933-
{
934-
return new FileInfo(path).Directory?.FullName;
935-
}
936-
catch (Exception exc)
937-
{
938-
logger.LogWarning($"Failed to get directory of '{path}': {exc}");
939-
}
940-
return null;
941-
}
942-
943847
private (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
944848
{
945849
var nugetConfigs = fileProvider.NugetConfigs;
@@ -981,7 +885,7 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
981885

982886
// Find feeds that are explicitly configured in the NuGet configuration files that we found.
983887
var explicitFeeds = nugetConfigs
984-
.SelectMany(config => GetFeeds(() => dotnet.GetNugetFeeds(config)))
888+
.SelectMany(config => feedManager.GetFeedsFromNugetConfig(config))
985889
.ToHashSet();
986890

987891
if (explicitFeeds.Count > 0)
@@ -995,10 +899,10 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
995899

996900
// If private package registries are configured for C#, then consider those
997901
// in addition to the ones that are configured in `nuget.config` files.
998-
if (hasPrivateRegistryFeeds)
902+
if (feedManager.HasPrivateRegistryFeeds)
999903
{
1000-
logger.LogInfo($"Found {privateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", privateRegistryFeeds.OrderBy(f => f))}");
1001-
explicitFeeds.UnionWith(privateRegistryFeeds);
904+
logger.LogInfo($"Found {feedManager.PrivateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", feedManager.PrivateRegistryFeeds.OrderBy(f => f))}");
905+
explicitFeeds.UnionWith(feedManager.PrivateRegistryFeeds);
1002906
}
1003907

1004908
HashSet<string> allFeeds = [];
@@ -1008,15 +912,15 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
1008912

1009913
// Obtain the list of feeds from the root source directory.
1010914
// If a NuGet file is present it will be respected, otherwise we will just get the machine/environment specific feeds.
1011-
var nugetFeedsFromRoot = GetFeeds(() => dotnet.GetNugetFeedsFromFolder(fileProvider.SourceDir.FullName));
915+
var nugetFeedsFromRoot = feedManager.GetFeedsFromFolder(fileProvider.SourceDir.FullName);
1012916
allFeeds.UnionWith(nugetFeedsFromRoot);
1013917

1014918
if (nugetConfigs.Count > 0)
1015919
{
1016920
var nugetConfigFeeds = nugetConfigs
1017-
.Select(GetDirectoryName)
921+
.Select(path => FileUtils.GetDirectoryName(path, logger))
1018922
.Where(folder => folder != null)
1019-
.SelectMany(folder => GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder!)))
923+
.SelectMany(folder => feedManager.GetFeedsFromFolder(folder!))
1020924
.ToHashSet();
1021925

1022926
allFeeds.UnionWith(nugetConfigFeeds);
@@ -1036,15 +940,12 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
1036940
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
1037941
private static partial Regex LegacyNugetPackage();
1038942

1039-
[GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
1040-
private static partial Regex EnabledNugetFeed();
1041-
1042943
public void Dispose()
1043944
{
1044945
PackageDirectory?.Dispose();
1045946
legacyPackageDirectory?.Dispose();
1046947
missingPackageDirectory?.Dispose();
1047-
emptyPackageDirectory?.Dispose();
948+
feedManager.Dispose();
1048949
}
1049950

1050951
/// <summary>

0 commit comments

Comments
 (0)