fix: Far more reliable downloads

fix: Account for program stopping mid download by writing to partial file, then renaming it
fix: Set a default maximum of 4 concurrent download tasks
fix: Don't check last changed headers for updates by default, reduces number of http calls when caching greatly
chore: better rclone command not found error message

Author: 0ffz

gradle.properties

@@ -1,4 +1,4 @@
 kotlin.code.style=official
 group=com.mineinabyss
-version=3.2.0-alpha.4
+version=3.2.0-alpha.5
 idofrontVersion=0.25.6

keepup-api/src/main/kotlin/com/mineinabyss/keepup/api/KeepupDownloader.kt

@@ -7,12 +7,11 @@ import com.mineinabyss.keepup.downloads.parsing.DownloadSource
 import com.mineinabyss.keepup.similarfiles.SimilarFileChecker
 import com.mineinabyss.keepup.type_checker.FileTypeChecker.SYSTEM_SUPPORTS_FILE
 import io.ktor.client.*
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.*
 import kotlinx.coroutines.channels.ReceiveChannel
 import kotlinx.coroutines.channels.produce
-import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
 import java.nio.file.Path
 import kotlin.io.path.absolute
 import kotlin.io.path.createDirectories
@@ -22,13 +21,17 @@ class KeepupDownloader(
     val http: HttpClient,
     val config: KeepupDownloaderConfig,
     val githubConfig: GithubConfig,
+    val downloadDispatcher: CoroutineDispatcher = Dispatchers.IO,
+    val maxConcurrentDownloads: Int = 4,
 ) {
+    private val concurrentDownloads = Semaphore(maxConcurrentDownloads)
+
     @OptIn(ExperimentalCoroutinesApi::class)
     fun download(
         vararg sources: DownloadSource,
         dest: Path,
         scope: CoroutineScope,
-    ): ReceiveChannel<DownloadResult> = scope.produce(Dispatchers.IO) {
+    ): ReceiveChannel<DownloadResult> = scope.produce {
         SYSTEM_SUPPORTS_FILE // check if system supports file command
         val similarFileChecker = if (config.ignoreSimilar) SimilarFileChecker(dest) else null
         val downloader = DownloadParser(
@@ -37,16 +40,16 @@ class KeepupDownloader(
             githubConfig = githubConfig,
             similarFileChecker = similarFileChecker,
         )
-
         sources.map { source ->
-            val downloadPathForKey = (config.downloadCache / source.keyInConfig).absolute()
-            downloadPathForKey.createDirectories()
-            launch {
-                downloader
-                    .download(source, downloadPathForKey)
-                    .forEach { channel.send(it) }
+            launch(downloadDispatcher) {
+                concurrentDownloads.withPermit {
+                    val downloadPathForKey = (config.downloadCache / source.keyInConfig).absolute()
+                    downloadPathForKey.createDirectories()
+                    downloader.download(source, downloadPathForKey)
+                        .forEach { channel.send(it) }
+                }
             }
-        }
+        }.joinAll()
     }
 }
 

keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/http/HttpDownloader.kt

@@ -19,29 +19,43 @@ class HttpDownloader(
     val source: DownloadSource,
     val targetDir: Path,
     val fileName: String = source.query.substringAfterLast("/"),
+    val overrideWhenHeadersChange: Boolean = false,
 ) : Downloader {
-    override suspend fun download(): List<DownloadResult> {
-        val cacheFile = targetDir.resolve("$fileName.cache")
-        val targetFile = targetDir.resolve(fileName)
+    suspend fun getCacheString(query: String): String {
         val headers = client.head(source.query)
         val length = headers.contentLength()
         val lastModified = headers.lastModified()
+        return "Last-Modified: $lastModified, Content-Length: $length"
+    }
 
-        val cache = "Last-Modified: $lastModified, Content-Length: $length"
-        if (targetFile.exists() && cacheFile.exists() && cacheFile.readText() == cache)
+    override suspend fun download(): List<DownloadResult> {
+        val cacheFile = targetDir.resolve("$fileName.cache")
+        val targetFile = targetDir.resolve(fileName)
+        val partial = targetDir.resolve("$fileName.partial")
+        val cacheString = if (overrideWhenHeadersChange) getCacheString(source.query) else null
+
+        // Check if target already exists and skip if it does, check last modified headers if overrideWhenHeadersChange is true
+        if (targetFile.exists() && (cacheString == null || (cacheFile.exists() && cacheFile.readText() == cacheString)))
             return listOf(DownloadResult.SkippedBecauseCached(targetFile, source.keyInConfig))
 
-        client.get(source.query) {
+        // Write to partial file, then move it to target once download is complete
+        partial.deleteIfExists()
+        client.prepareGet {
+            url(source.query)
             timeout {
                 requestTimeoutMillis = 30.seconds.inWholeMilliseconds
             }
+        }.execute {
+            it.bodyAsChannel()
+                .copyAndClose(partial.toFile().writeChannel())
         }
-            .bodyAsChannel()
-            .copyAndClose(targetFile.toFile().writeChannel())
+
+        targetFile.deleteIfExists()
+        partial.moveTo(targetFile)
 
         // Only mark as cached after download is complete
         cacheFile.deleteIfExists()
-        cacheFile.createFile().writeText(cache)
+        cacheFile.createFile().writeText(cacheString ?: getCacheString(source.query))
 
         return listOf(DownloadResult.Downloaded(targetFile, source.keyInConfig))
     }

keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/rclone/RcloneDownloader.kt

@@ -1,5 +1,6 @@
 package com.mineinabyss.keepup.downloads.rclone
 
+import com.lordcodes.turtle.ShellCommandNotFoundException
 import com.mineinabyss.keepup.downloads.DownloadResult
 import com.mineinabyss.keepup.downloads.Downloader
 import com.mineinabyss.keepup.downloads.parsing.DownloadSource
@@ -11,7 +12,12 @@ class RcloneDownloader(
     val targetDir: Path,
 ) : Downloader {
     override suspend fun download(): List<DownloadResult> {
-        val downloadPath = Rclone.sync(source.query, targetDir)
+        val downloadPath = runCatching { Rclone.sync(source.query, targetDir) }
+            .onFailure {
+                if (it is ShellCommandNotFoundException)
+                    return listOf(DownloadResult.Failure("rclone command not found", source.keyInConfig))
+            }
+            .getOrThrow()
         return listOf(DownloadResult.Downloaded(downloadPath, source.keyInConfig, overrideInfoMsg = MSG.rclone))
     }
 }