Skip to content

Commit 8adb4eb

Browse files
PathoschildPathoschild
committed
Merge branch 'add-stats-page' into develop
2 parents 13818ab + 012f1cd commit 8adb4eb

15 files changed

Lines changed: 1855 additions & 14 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ _ReSharper*/
3131
# sensitive files
3232
appsettings.Development.json
3333

34-
# Azure generated files
34+
# auto-generated files
35+
src/SMAPI.Web/wwwroot/Content/data
3536
src/SMAPI.Web/Properties/PublishProfiles
3637
src/SMAPI.Web/Properties/ServiceDependencies
3738

‎src/SMAPI.Web/BackgroundService.cs‎

Lines changed: 54 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
using System;
22
using System.Diagnostics.CodeAnalysis;
3+
using System.IO;
34
using System.Threading;
45
using System.Threading.Tasks;
56
using Hangfire;
67
using Hangfire.Console;
78
using Hangfire.Server;
89
using Humanizer;
10+
using Microsoft.AspNetCore.Hosting;
911
using Microsoft.Extensions.Hosting;
1012
using Microsoft.Extensions.Options;
1113
using StardewModdingAPI.Toolkit;
@@ -17,6 +19,7 @@
1719
using StardewModdingAPI.Web.Framework.Caching;
1820
using StardewModdingAPI.Web.Framework.Caching.CompatibilityRepo;
1921
using StardewModdingAPI.Web.Framework.Caching.CurseForgeExport;
22+
using StardewModdingAPI.Web.Framework.Caching.ModDataset;
2023
using StardewModdingAPI.Web.Framework.Caching.ModDropExport;
2124
using StardewModdingAPI.Web.Framework.Caching.Mods;
2225
using StardewModdingAPI.Web.Framework.Caching.NexusExport;
@@ -64,6 +67,12 @@ internal class BackgroundService : IHostedService, IDisposable
6467
/// <summary>The config settings for mod update checks.</summary>
6568
private static IOptions<ModUpdateCheckConfig>? UpdateCheckConfig;
6669

70+
/// <summary>The mod dataset repository.</summary>
71+
private static IModDatasetRepository? ModDatasetRepo;
72+
73+
/// <summary>The web root path for writing static data files.</summary>
74+
private static string? WebRootPath;
75+
6776
/// <summary>Whether the service has been started.</summary>
6877
[MemberNotNullWhen(true,
6978
nameof(BackgroundService.JobServer),
@@ -75,7 +84,9 @@ internal class BackgroundService : IHostedService, IDisposable
7584
nameof(BackgroundService.ModDropExportCache),
7685
nameof(BackgroundService.NexusExportApiClient),
7786
nameof(BackgroundService.NexusExportCache),
78-
nameof(BackgroundService.UpdateCheckConfig)
87+
nameof(BackgroundService.ModDatasetRepo),
88+
nameof(BackgroundService.UpdateCheckConfig),
89+
nameof(BackgroundService.WebRootPath)
7990
)]
8091
private static bool IsStarted { get; set; }
8192

@@ -90,16 +101,18 @@ internal class BackgroundService : IHostedService, IDisposable
90101
** Hosted service
91102
****/
92103
/// <summary>Construct an instance.</summary>
93-
/// <param name="compatibilityCache">The cache in which to store compatibility list data.</param>
94-
/// <param name="modCache">The cache in which to store mod data.</param>
95-
/// <param name="curseForgeExportCache">The cache in which to store mod data from the CurseForge export API.</param>
96-
/// <param name="curseForgeExportApiClient">The HTTP client for fetching the mod export from the CurseForge export API.</param>
97-
/// /// <param name="modDropExportCache">The cache in which to store mod data from the ModDrop export API.</param>
98-
/// <param name="modDropExportApiClient">The HTTP client for fetching the mod export from the ModDrop export API.</param>
99-
/// <param name="nexusExportCache">The cache in which to store mod data from the Nexus export API.</param>
100-
/// <param name="nexusExportApiClient">The HTTP client for fetching the mod export from the Nexus Mods export API.</param>
104+
/// <param name="compatibilityCache"><inheritdoc cref="CompatibilityCache" path="/summary"/></param>
105+
/// <param name="modCache"><inheritdoc cref="ModCache" path="/summary"/></param>
106+
/// <param name="curseForgeExportCache"><inheritdoc cref="CurseForgeExportCache" path="/summary"/></param>
107+
/// <param name="curseForgeExportApiClient"><inheritdoc cref="CurseForgeExportApiClient" path="/summary"/></param>
108+
/// <param name="modDropExportCache"><inheritdoc cref="ModDropExportCache" path="/summary"/></param>
109+
/// <param name="modDropExportApiClient"><inheritdoc cref="ModDropExportApiClient" path="/summary"/></param>
110+
/// <param name="nexusExportCache"><inheritdoc cref="NexusExportCache" path="/summary"/></param>
111+
/// <param name="nexusExportApiClient"><inheritdoc cref="NexusExportApiClient" path="/summary"/></param>
112+
/// <param name="modDatasetRepo"><inheritdoc cref="ModDatasetRepo" path="/summary"/></param>
101113
/// <param name="hangfireStorage">The Hangfire storage implementation.</param>
102-
/// <param name="updateCheckConfig">The config settings for mod update checks.</param>
114+
/// <param name="updateCheckConfig"><inheritdoc cref="UpdateCheckConfig" path="/summary"/></param>
115+
/// <param name="hostEnvironment">The web host environment metadata.</param>
103116
[SuppressMessage("ReSharper", "UnusedParameter.Local", Justification = "The Hangfire reference forces it to initialize first, since it's needed by the background service.")]
104117
public BackgroundService(
105118
ICompatibilityCacheRepository compatibilityCache,
@@ -110,8 +123,10 @@ public BackgroundService(
110123
IModDropExportApiClient modDropExportApiClient,
111124
INexusExportCacheRepository nexusExportCache,
112125
INexusExportApiClient nexusExportApiClient,
126+
IModDatasetRepository modDatasetRepo,
113127
JobStorage hangfireStorage,
114-
IOptions<ModUpdateCheckConfig> updateCheckConfig
128+
IOptions<ModUpdateCheckConfig> updateCheckConfig,
129+
IWebHostEnvironment hostEnvironment
115130
)
116131
{
117132
BackgroundService.CompatibilityCache = compatibilityCache;
@@ -122,7 +137,9 @@ IOptions<ModUpdateCheckConfig> updateCheckConfig
122137
BackgroundService.ModDropExportCache = modDropExportCache;
123138
BackgroundService.NexusExportCache = nexusExportCache;
124139
BackgroundService.NexusExportApiClient = nexusExportApiClient;
140+
BackgroundService.ModDatasetRepo = modDatasetRepo;
125141
BackgroundService.UpdateCheckConfig = updateCheckConfig;
142+
BackgroundService.WebRootPath = hostEnvironment.WebRootPath;
126143

127144
_ = hangfireStorage; // parameter is only received to initialize it before the background service
128145
}
@@ -146,6 +163,7 @@ public Task StartAsync(CancellationToken cancellationToken)
146163
if (enableNexusExport)
147164
BackgroundJob.Enqueue(() => BackgroundService.UpdateNexusExportAsync(null));
148165
BackgroundJob.Enqueue(() => BackgroundService.RemoveStaleModsAsync());
166+
BackgroundJob.Enqueue(() => BackgroundService.UpdateModDatasetAsync(null));
149167

150168
// set recurring tasks
151169
RecurringJob.AddOrUpdate("update compatibility list", () => BackgroundService.UpdateCompatibilityListAsync(null), "*/10 * * * *"); // every 10 minutes
@@ -156,6 +174,7 @@ public Task StartAsync(CancellationToken cancellationToken)
156174
if (enableNexusExport)
157175
RecurringJob.AddOrUpdate("update Nexus export", () => BackgroundService.UpdateNexusExportAsync(null), "*/10 * * * *");
158176
RecurringJob.AddOrUpdate("remove stale mods", () => BackgroundService.RemoveStaleModsAsync(), "2/10 * * * *"); // offset by 2 minutes so it runs after updates (e.g. 00:02, 00:12, etc)
177+
RecurringJob.AddOrUpdate("update mod dataset", () => BackgroundService.UpdateModDatasetAsync(null), "0 * * * *"); // hourly
159178

160179
BackgroundService.IsStarted = true;
161180

@@ -242,6 +261,30 @@ await UpdateExportAsync(
242261
);
243262
}
244263

264+
/// <summary>Clone or pull the Stardew mod dataset repo, then copy the latest data files into the web root for use by frontend scripts.</summary>
265+
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
266+
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])]
267+
public static async Task UpdateModDatasetAsync(PerformContext? context)
268+
{
269+
if (!BackgroundService.IsStarted)
270+
throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
271+
272+
// update repo
273+
context.WriteLine("Updating mod dataset repo...");
274+
await BackgroundService.ModDatasetRepo.UpdateAsync(context.WriteLine);
275+
276+
// copy files
277+
context.WriteLine("Copying data files for script use...");
278+
string copyToPath = Path.Combine(BackgroundService.WebRootPath, "Content", "data", "mods-by-type.jsonl");
279+
Directory.CreateDirectory(Path.GetDirectoryName(copyToPath)!);
280+
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("stats/mods by type.jsonl"), copyToPath, overwrite: true);
281+
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("stats/Content Patcher packs by format version.jsonl"), Path.Combine(BackgroundService.WebRootPath, "Content", "data", "content-packs-by-format.jsonl"), overwrite: true);
282+
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("reference-data/SMAPI costs.jsonl"), Path.Combine(BackgroundService.WebRootPath, "Content", "data", "smapi-costs.jsonl"), overwrite: true);
283+
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("reference-data/SMAPI DNS queries.json"), Path.Combine(BackgroundService.WebRootPath, "Content", "data", "smapi-dns-queries.json"), overwrite: true);
284+
285+
context.WriteLine("Done!");
286+
}
287+
245288
/// <summary>Remove mods which haven't been requested in over 48 hours.</summary>
246289
public static Task RemoveStaleModsAsync()
247290
{

‎src/SMAPI.Web/Controllers/IndexController.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,13 @@ public ViewResult Privacy()
7777
return this.View();
7878
}
7979

80+
/// <summary>Display the stats page.</summary>
81+
[HttpGet("/stats")]
82+
public ViewResult Stats()
83+
{
84+
return this.View();
85+
}
86+
8087

8188
/*********
8289
** Private methods

‎src/SMAPI.Web/Controllers/ModsController.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Linq;
3-
using System.Text.RegularExpressions;
43
using System.Threading.Tasks;
54
using Microsoft.AspNetCore.Mvc;
65
using Microsoft.Extensions.Options;

‎src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using System;
2+
using System.Threading.Tasks;
3+
4+
namespace StardewModdingAPI.Web.Framework.Caching.ModDataset;
5+
6+
/// <summary>Manages the Stardew mod dataset repo.</summary>
7+
internal interface IModDatasetRepository
8+
{
9+
/*********
10+
** Methods
11+
*********/
12+
/// <summary>Fetch the latest mod dataset, if it changed or hasn't been fetched yet.</summary>
13+
/// <param name="log">A callback which should receive progress messages for logging.</param>
14+
Task UpdateAsync(Action<string>? log = null);
15+
16+
/// <summary>Get the full path to a file in the mod dataset.</summary>
17+
/// <param name="relativePath">The file path relative to the <c>dataset</c> folder in the mod dataset repo.</param>
18+
string GetFilePath(string relativePath);
19+
}

‎src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
using System;
2+
using System.Diagnostics;
3+
using System.IO;
4+
using System.Threading.Tasks;
5+
6+
namespace StardewModdingAPI.Web.Framework.Caching.ModDataset;
7+
8+
/// <inheritdoc cref="IModDatasetRepository" />
9+
internal class ModDatasetRepository : IModDatasetRepository
10+
{
11+
/*********
12+
** Fields
13+
*********/
14+
/// <summary>The HTTPS URL of the mod dataset Git repo.</summary>
15+
private readonly string RepoUrl;
16+
17+
/// <summary>The full path to the mod dataset repo.</summary>
18+
private readonly string LocalRepoPath;
19+
20+
21+
/*********
22+
** Public methods
23+
*********/
24+
/// <summary>Construct an instance.</summary>
25+
/// <param name="repoUrl"><inheritdoc cref="RepoUrl" path="/summary" /></param>
26+
/// <param name="localRepoPath">The path to the mod dataset repo.</param>
27+
public ModDatasetRepository(string repoUrl, string localRepoPath)
28+
{
29+
this.RepoUrl = repoUrl;
30+
this.LocalRepoPath = Path.GetFullPath(
31+
Environment.ExpandEnvironmentVariables(localRepoPath)
32+
);
33+
}
34+
35+
/// <inheritdoc />
36+
public async Task UpdateAsync(Action<string>? log = null)
37+
{
38+
if (!Directory.Exists(Path.Combine(this.LocalRepoPath, ".git")))
39+
{
40+
log?.Invoke("Cloning mod dataset repo...");
41+
Directory.CreateDirectory(Path.GetDirectoryName(this.LocalRepoPath)!);
42+
await this.RunGitAsync("clone", "--depth", "1", this.RepoUrl, this.LocalRepoPath);
43+
}
44+
else
45+
{
46+
log?.Invoke("Fetching latest changes from mod dataset repo...");
47+
await this.RunGitAsync("-C", this.LocalRepoPath, "fetch", "--depth", "1", "origin");
48+
await this.RunGitAsync("-C", this.LocalRepoPath, "reset", "--hard", "FETCH_HEAD");
49+
}
50+
}
51+
52+
/// <inheritdoc />
53+
public string GetFilePath(string relativePath)
54+
{
55+
return Path.Combine(this.LocalRepoPath, "dataset", relativePath);
56+
}
57+
58+
59+
/*********
60+
** Private methods
61+
*********/
62+
/// <summary>Run a Git command and wait for it to complete.</summary>
63+
/// <param name="args">The git arguments.</param>
64+
/// <exception cref="InvalidOperationException">The Git command exited with a non-zero exit code.</exception>
65+
private async Task RunGitAsync(params string[] args)
66+
{
67+
ProcessStartInfo startInfo = new("git", args)
68+
{
69+
RedirectStandardOutput = true,
70+
RedirectStandardError = true,
71+
UseShellExecute = false,
72+
CreateNoWindow = true
73+
};
74+
75+
using Process process = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start Git process.");
76+
77+
Task readOutput = process.StandardOutput.ReadToEndAsync();
78+
Task<string> readError = process.StandardError.ReadToEndAsync();
79+
await process.WaitForExitAsync();
80+
await Task.WhenAll(readOutput, readError);
81+
82+
if (process.ExitCode != 0)
83+
throw new InvalidOperationException($"Git command failed with exit code {process.ExitCode}: {await readError}.");
84+
}
85+
}

‎src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
namespace StardewModdingAPI.Web.Framework.ConfigModels;
2+
3+
/// <summary>The config settings for the Stardew mod dataset repo.</summary>
4+
internal class ModDatasetConfig
5+
{
6+
/*********
7+
** Accessors
8+
*********/
9+
/// <summary>The HTTPS URL of the mod dataset Git repo.</summary>
10+
public string RepoUrl { get; set; } = null!;
11+
12+
/// <summary>The local path into which to clone the mod dataset repo.</summary>
13+
public string LocalPath { get; set; } = null!;
14+
}

‎src/SMAPI.Web/Startup.cs‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
using StardewModdingAPI.Web.Framework;
2323
using StardewModdingAPI.Web.Framework.Caching.CompatibilityRepo;
2424
using StardewModdingAPI.Web.Framework.Caching.CurseForgeExport;
25+
using StardewModdingAPI.Web.Framework.Caching.ModDataset;
2526
using StardewModdingAPI.Web.Framework.Caching.ModDropExport;
2627
using StardewModdingAPI.Web.Framework.Caching.Mods;
2728
using StardewModdingAPI.Web.Framework.Caching.NexusExport;
@@ -108,6 +109,12 @@ public void ConfigureServices(IServiceCollection services)
108109
services.AddSingleton<IModDropExportCacheRepository>(new ModDropExportCacheMemoryRepository());
109110
services.AddSingleton<INexusExportCacheRepository>(new NexusExportCacheMemoryRepository());
110111

112+
// init mod dataset
113+
{
114+
ModDatasetConfig config = this.Configuration.GetRequiredSection("ModDataset").Get<ModDatasetConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded.");
115+
services.AddSingleton<IModDatasetRepository>(new ModDatasetRepository(config.RepoUrl, config.LocalPath));
116+
}
117+
111118
// init Hangfire
112119
services
113120
.AddHangfire((_, config) =>
@@ -245,7 +252,17 @@ public void Configure(IApplicationBuilder app)
245252
.WithOrigins("https://smapi.io")
246253
)
247254
.UseRewriter(this.GetRedirectRules())
248-
.UseStaticFiles(new StaticFileOptions { OnPrepareResponse = this.OnPrepareStaticFileResponse }) // wwwroot folder
255+
.UseStaticFiles(
256+
// serve static files in wwwroot folder
257+
new StaticFileOptions
258+
{
259+
OnPrepareResponse = this.OnPrepareStaticFileResponse,
260+
ContentTypeProvider = new FileExtensionContentTypeProvider
261+
{
262+
Mappings = { [".jsonl"] = "application/jsonl" }
263+
}
264+
}
265+
)
249266
.UseRouting()
250267
.UseAuthorization()
251268
.UseEndpoints(p =>

0 commit comments

Comments
 (0)