Set up integration tests for DEEPWELL

.github/workflows/deepwell.yaml

@@ -19,10 +19,48 @@ on:
 
 jobs:
   binary_build_and_test:
-    name: Executable
+    name: Test
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      valkey:
+        image: valkey/valkey:alpine
+        options: >-
+          --health-cmd "valkey-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      minio:
+        image: minio/minio:latest
+        env:
+          MINIO_API_PORT_NUMBER: '9000'
+          MINIO_SERVER_ACCESS_KEY: 's3-access-key'
+          MINIO_SERVER_SECRET_KEY: 's3-secret-key'
+          MINIO_REGION_NAME: 'local'
+          INITIAL_BUCKETS: 'deepwell-files deepwell-text-blocks'
+        options: >-
+          --health-cmd "bash -c ':> /dev/tcp/127.0.0.1/9000'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     env:
       RUSTFLAGS: -D warnings
+      DATABASE_URL: postgres://postgres:postgres@postgres/postgres
+      REDIS_URL: redis://valkey
+      S3_CUSTOM_ENDPOINT: http://minio:9000
+      S3_ACCESS_KEY_ID: 's3-access-key'
+      S3_SECRET_ACCESS_KEY: 's3-secret-key'
+
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -50,6 +88,11 @@ jobs:
         env:
           DEEPWELL_RUNTIME_ACTION: validate-config
 
+      - name: Run Seeder
+        run: cd deepwell && cargo run -- config.example.toml
+        env:
+          DEEPWELL_RUNTIME_ACTION: run-seeder
+
       - name: Test
         run: cd deepwell && cargo test --all-features -- --nocapture --test-threads 1
 

deepwell/src/config/mod.rs

@@ -37,8 +37,13 @@ pub struct SetupConfig {
 }
 
 impl SetupConfig {
-    pub fn load() -> Self {
-        run_runtime_action();
+    pub async fn load() -> Self {
+        run_runtime_action().await;
+        Self::load2()
+    }
+
+    /// Internal version of `load()` that doesn't do runtime actions.
+    pub(crate) fn load2() -> Self {
         let secrets = Secrets::load();
         let config = parse_args();
         SetupConfig { secrets, config }

deepwell/src/config/object.rs

@@ -261,4 +261,70 @@ impl Config {
                 .display(),
         );
     }
+
+    /// Configuration file for integration testing.
+    #[allow(dead_code)]
+    pub fn integration_testing() -> Self {
+        Config {
+            raw_toml: str!("[[ BUILT-IN ]]"),
+            raw_toml_path: PathBuf::from("/dev/null"),
+            logger: true,
+            logger_level: LevelFilter::Debug,
+            address: "[::1]:2747".parse().unwrap(),
+            pid_file: None,
+            main_domain: str!(".wikijump.com"),
+            main_domain_no_dot: str!("wikijump.com"),
+            files_domain: str!(".wjfiles.com"),
+            files_domain_no_dot: str!("wjfiles.com"),
+            watch_files: false,
+            run_seeder: false,
+            seeder_path: PathBuf::from("seeder"),
+            localization_path: PathBuf::from("../locales"),
+            authentication_fail_delay: StdDuration::from_millis(1),
+            session_token_prefix: str!("wj:"),
+            session_token_length: 16,
+            normal_session_duration: time::Duration::minutes(30),
+            restricted_session_duration: time::Duration::minutes(5),
+            recovery_code_count: 4,
+            recovery_code_length: 8,
+            totp_time_step: 30,
+            totp_time_skew: 1,
+            job_workers: NonZeroU16::new(2).unwrap(),
+            job_max_attempts: 1,
+            job_work_delay: StdDuration::from_millis(1),
+            job_min_poll_delay: StdDuration::from_millis(1),
+            job_max_poll_delay: StdDuration::from_millis(500),
+            job_prune_session: StdDuration::from_secs(60),
+            job_prune_uploads: StdDuration::from_secs(60),
+            job_prune_text: StdDuration::from_secs(60),
+            job_name_change_refill: StdDuration::from_secs(60),
+            job_lift_expired_punishments: StdDuration::from_secs(60),
+            preprocess_timeout: StdDuration::from_millis(500),
+            render_timeout: StdDuration::from_millis(1000),
+            rerender_skip: vec![
+                (1, Some(time::Duration::milliseconds(100))),
+                (5, Some(time::Duration::milliseconds(500))),
+                (10, None),
+            ],
+            message_layout: Layout::Wikijump,
+            default_page_layout: Layout::Wikidot,
+            special_page_prefix: str!("_"),
+            special_page_template: str!("_template"),
+            special_page_missing: str!("_404"),
+            special_page_private: str!("_public"),
+            special_page_banned: str!("_ban"),
+            default_name_changes: 2,
+            maximum_name_changes: 2,
+            refill_name_change: None,
+            minimum_name_bytes: 4,
+            minimum_name_chars: 2,
+            presigned_path_length: 4,
+            presigned_expiry_secs: 60,
+            maximum_blob_size: 512 * 1024,
+            maximum_avatar_size: 250 * 1024,
+            maximum_message_subject_bytes: 128,
+            maximum_message_body_bytes: 10_000,
+            maximum_message_recipients: 3,
+        }
+    }
 }

deepwell/src/config/runtime_action.rs

@@ -24,11 +24,12 @@
 //! not run as a daemon, but instead perform a special action or check,
 //! as if motivated by a script.
 
-use super::Config;
+use super::{Config, SetupConfig};
+use crate::{api, database};
 use std::path::PathBuf;
 use std::{env, process};
 
-pub fn run_runtime_action() {
+pub async fn run_runtime_action() {
     // Get action name, if specified.
     // Otherwise return and perform normal execution.
     let Ok(action_name) = env::var("DEEPWELL_RUNTIME_ACTION") else {
@@ -38,6 +39,7 @@ pub fn run_runtime_action() {
     // Run appropriate runtime action.
     let return_code = match action_name.as_str() {
         "config" | "validate-config" => validate_config(),
+        "seeder" | "run-seeder" => run_seeder().await,
         _ => {
             eprintln!("Unknown runtime action: {action_name}");
             process::exit(1);
@@ -69,3 +71,30 @@ fn validate_config() -> i32 {
     println!("All config files have been checked, {return_code} failed");
     return_code
 }
+
+async fn run_seeder() -> i32 {
+    println!("Running action: Database seeder");
+
+    let SetupConfig { secrets, config } = SetupConfig::load2();
+
+    let app_state = match api::build_server_state(config, secrets).await {
+        Ok(app_state) => app_state,
+        Err(error) => {
+            println!("error");
+            eprintln!("{error}");
+            return 1;
+        }
+    };
+
+    match database::seed(&app_state).await {
+        Ok(()) => {
+            println!("success");
+            0
+        }
+        Err(error) => {
+            println!("error");
+            eprintln!("{error}");
+            1
+        }
+    }
+}

deepwell/src/endpoints/info.rs

@@ -28,33 +28,33 @@ use time::OffsetDateTime;
 
 #[derive(Serialize, Debug, Clone)]
 pub struct Info {
-    package: PackageInfo,
-    compile_info: CompileInfo,
+    pub package: PackageInfo,
+    pub compile_info: CompileInfo,
 
     #[serde(with = "time::serde::rfc3339")]
-    current_time: OffsetDateTime,
-    hostname: &'static str,
-    config_path: PathBuf,
+    pub current_time: OffsetDateTime,
+    pub hostname: &'static str,
+    pub config_path: PathBuf,
 }
 
 #[derive(Serialize, Debug, Clone)]
 pub struct PackageInfo {
-    name: &'static str,
-    description: &'static str,
-    license: &'static str,
-    repository: &'static str,
-    version: &'static str,
+    pub name: &'static str,
+    pub description: &'static str,
+    pub license: &'static str,
+    pub repository: &'static str,
+    pub version: &'static str,
 }
 
 #[derive(Serialize, Debug, Clone)]
 pub struct CompileInfo {
     #[serde(with = "time::serde::rfc3339")]
-    built_at: OffsetDateTime,
-    rustc_version: &'static str,
-    endian: &'static str,
-    target: &'static str,
-    threads: u32,
-    git_commit: Option<&'static str>,
+    pub built_at: OffsetDateTime,
+    pub rustc_version: &'static str,
+    pub endian: &'static str,
+    pub target: &'static str,
+    pub threads: u32,
+    pub git_commit: Option<&'static str>,
 }
 
 pub async fn server_info(

deepwell/src/endpoints/locale.rs

@@ -26,10 +26,10 @@ use unic_langid::LanguageIdentifier;
 
 #[derive(Serialize, Debug, Clone)]
 pub struct LocaleOutput {
-    language: String,
-    script: Option<String>,
-    region: Option<String>,
-    variants: Vec<String>,
+    pub language: String,
+    pub script: Option<String>,
+    pub region: Option<String>,
+    pub variants: Vec<String>,
 }
 
 #[derive(Deserialize, Debug, Clone)]

deepwell/src/lib.rs

@@ -0,0 +1,57 @@
+/*
+ * lib.rs
+ *
+ * DEEPWELL - Wikijump API provider and database manager
+ * Copyright (C) 2019-2025 Wikijump Team
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//! Library version of the deepwell crate.
+//!
+//! This exists only to enable integration test runners to run the deepwell
+//! server. It should have the largely same modules as `main.rs`.
+
+#[macro_use]
+extern crate log;
+
+#[macro_use]
+extern crate futures;
+
+#[macro_use]
+extern crate serde;
+
+#[macro_use]
+extern crate str_macro;
+
+#[macro_use]
+mod macros;
+
+// Auto-generated by sea-orm-cli
+#[allow(unused_imports)]
+mod models;
+
+pub mod api;
+pub mod config;
+pub mod constants;
+pub mod database;
+pub mod endpoints;
+pub mod hash;
+pub mod info;
+pub mod license;
+pub mod locales;
+pub mod redis;
+pub mod services;
+pub mod types;
+pub mod utils;

deepwell/src/main.rs

@@ -73,7 +73,7 @@ use std::process;
 #[tokio::main]
 async fn main() -> Result<()> {
     // Load the configuration so we can set up
-    let SetupConfig { secrets, config } = SetupConfig::load();
+    let SetupConfig { secrets, config } = SetupConfig::load().await;
     let address = config.address;
     let run_seeder = config.run_seeder;
 

deepwell/src/services/blob/service.rs

@@ -287,10 +287,9 @@ impl BlobService {
     /// Helper function to do the actual "move" step of blob finalization.
     /// This is where, after uploading to the presign URL, the S3 object is
     /// then moved to its permanent location with a hashed name.
-    ///
-    /// NOTE: Because S3 changes cannot be rolled back on error, we are
-    ///       creating a separate transaction here so that `blob_pending`
-    ///       changes are persistent even if the outer request fails.
+    // NOTE: Because S3 changes cannot be rolled back on error, we are
+    //       creating a separate transaction here so that `blob_pending`
+    //       changes are persistent even if the outer request fails.
     async fn move_uploaded(
         ctx: &ServiceContext<'_>,
         pending_blob_id: &str,

deepwell/src/services/caddy/service.rs

@@ -22,13 +22,12 @@
 //!
 //! This is primarily concerned with generating the `Caddyfile` that
 //! powers the server, which is where host → site mapping is performed.
-//!
-//! NOTE: This file contains hard tabs, as this is what we want to use for
-//!       `Caddyfile` generation. If you're opening this file, mind the git
-//!       diff!
-//!
-//!       If your editor munges the tabs please discard those changes.
-//!       Remember, `git add -p` is your friend!
+// NOTE: This file contains hard tabs, as this is what we want to use for
+//       Caddyfile generation. If you're opening this file, mind the git
+//       diff!
+//
+// If your editor munges the tabs please discard those changes.
+// Remember, "git add -p" is your friend!
 
 use super::prelude::*;
 use crate::models::alias::Model as AliasModel;

deepwell/src/services/caddy/test.rs

@@ -157,7 +157,7 @@ macro_rules! test_file_path {
     ($suffix:expr) => {
         concat!(
             env!("CARGO_MANIFEST_DIR"),
-            "/test/caddy/Caddyfile.",
+            "/tests/caddy/Caddyfile.",
             $suffix,
         )
     };

deepwell/src/utils/string.rs

@@ -90,7 +90,7 @@ pub fn trim_spaces_in_place(string: &mut String) {
 /// these characters. We don't remove these characters from all locale strings
 /// since they are a desired localization property of Fluent.
 ///
-/// See https://fluent-compiler.readthedocs.io/en/latest/usage.html#:~:text=You%20will%20notice%20the%20extra%20characters%20\u2068%20and%20\u2069%20in%20the%20output.
+/// See the [Fluent Docs](https://fluent-compiler.readthedocs.io/en/latest/usage.html#:~:text=You%20will%20notice%20the%20extra%20characters%20\u2068%20and%20\u2069%20in%20the%20output.)
 #[inline]
 pub fn strip_fluent_control_chars(string: &mut String) {
     static CONTROL_CHAR_REGEX: LazyLock<Regex> =