diff --git a/config/default.yaml b/config/default.yaml index b33871bdf8e..db2c208b4f8 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -884,6 +884,15 @@ import: proxies: # - "https://username:password@example.com:8888" + cookies: + # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited + # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported + # That account may be rate limited or temporarily blocked + # PeerTube expects a Netscape-format file at storage/tmp-persistent/youtube-cookies.txt + # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions + # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup + enabled: false + # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) torrent: # We recommend to only enable magnet URI/torrent import if you trust your users diff --git a/config/production.yaml.example b/config/production.yaml.example index c37b64e2da3..92655d91462 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -894,6 +894,15 @@ import: proxies: # - "https://username:password@example.com:8888" + cookies: + # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited + # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported + # That account may be rate limited or temporarily blocked + # PeerTube expects a Netscape-format file at /var/www/peertube/storage/tmp-persistent/youtube-cookies.txt + # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions + # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup + enabled: false + # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) torrent: # We recommend to only enable magnet URI/torrent import if you trust your users diff --git a/packages/tests/src/server-helpers/youtube-dl.ts b/packages/tests/src/server-helpers/youtube-dl.ts index f4922d5e700..25451f83644 100644 --- a/packages/tests/src/server-helpers/youtube-dl.ts +++ b/packages/tests/src/server-helpers/youtube-dl.ts @@ -1,7 +1,11 @@ /* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' +import { mkdtemp, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' import { YoutubeDLCLI } from '@peertube/peertube-server/core/helpers/youtube-dl/youtube-dl-cli.js' +import { logger } from '@peertube/peertube-server/core/helpers/logger.js' import { CONFIG } from '@peertube/peertube-server/core/initializers/config.js' describe('YoutubeDLCLI', function () { @@ -91,4 +95,96 @@ describe('YoutubeDLCLI', function () { } }) }) + + describe('wrapWithCookiesOptions', function () { + let cli: any + + before(function () { + cli = Object.create(YoutubeDLCLI.prototype) + }) + + it('Should prepend cookies file when configured and the file exists', async function () { + const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR') + const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED') + const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-')) + const cookiesFile = join(tempDir, 'youtube-cookies.txt') + + await writeFile(cookiesFile, '# Netscape HTTP Cookie File\n') + + Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', { + get: () => tempDir, + configurable: true + }) + + Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', { + get: () => true, + configurable: true + }) + + try { + const inputArgs = [ '--dump-json', '-f', 'best' ] + const result: string[] = await cli.wrapWithCookiesOptions(inputArgs) + + expect(result).to.deep.equal([ '--cookies', cookiesFile, ...inputArgs ]) + } finally { + Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor) + Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor) + await rm(tempDir, { recursive: true, force: true }) + } + }) + + it('Should log an error and continue when the cookies file is missing', async function () { + const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR') + const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED') + const originalLoggerError = logger.error + const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-')) + const loggedMessages: any[][] = [] + + Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', { + get: () => tempDir, + configurable: true + }) + + Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', { + get: () => true, + configurable: true + }) + + ;(logger as any).error = (...args: any[]) => { + loggedMessages.push(args) + } + + try { + const inputArgs = [ '--dump-json', '-f', 'best' ] + const result: string[] = await cli.wrapWithCookiesOptions(inputArgs) + + expect(result).to.deep.equal(inputArgs) + expect(loggedMessages).to.have.lengthOf(1) + expect(loggedMessages[0][0]).to.contain('yt-dlp cookies are enabled but the cookies file %s does not exist') + } finally { + Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor) + Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor) + ;(logger as any).error = originalLoggerError + await rm(tempDir, { recursive: true, force: true }) + } + }) + + it('Should not modify args when cookies are disabled', async function () { + const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED') + + Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', { + get: () => false, + configurable: true + }) + + try { + const inputArgs = [ '--dump-json', '-f', 'best' ] + const result: string[] = await cli.wrapWithCookiesOptions(inputArgs) + + expect(result).to.deep.equal(inputArgs) + } finally { + Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor) + } + }) + }) }) diff --git a/server/core/helpers/youtube-dl/youtube-dl-cli.ts b/server/core/helpers/youtube-dl/youtube-dl-cli.ts index 299a3be988a..3a6e2f7251c 100644 --- a/server/core/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/core/helpers/youtube-dl/youtube-dl-cli.ts @@ -215,6 +215,7 @@ export class YoutubeDLCLI { let completeArgs = this.wrapWithJSRuntimeOptions(args) completeArgs = this.wrapWithProxyOptions(completeArgs) + completeArgs = await this.wrapWithCookiesOptions(completeArgs) completeArgs = this.wrapWithIPOptions(completeArgs) completeArgs = this.wrapWithFFmpegOptions(completeArgs) @@ -272,6 +273,28 @@ export class YoutubeDLCLI { return args } + private async wrapWithCookiesOptions (args: string[]) { + if (!CONFIG.IMPORT.VIDEOS.HTTP.COOKIES.ENABLED) { + return args + } + + const cookiesPath = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, 'youtube-cookies.txt') + + if (!await pathExists(cookiesPath)) { + logger.error( + 'yt-dlp cookies are enabled but the cookies file %s does not exist. Continuing without cookies.', + cookiesPath, + lTags() + ) + + return args + } + + logger.debug('Using cookies file %s for YoutubeDL', cookiesPath, lTags()) + + return [ '--cookies', cookiesPath ].concat(args) + } + private wrapWithFFmpegOptions (args: string[]) { if (process.env.FFMPEG_PATH) { logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags()) diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts index 54e338e106d..b8c45fe2791 100644 --- a/server/core/initializers/checker-before-init.ts +++ b/server/core/initializers/checker-before-init.ts @@ -119,6 +119,7 @@ export function checkMissedConfig () { 'import.videos.timeout', 'import.videos.http.force_ipv4', 'import.videos.http.proxies', + 'import.videos.http.cookies.enabled', 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', 'import.video_channel_synchronization.check_interval', diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index 49fecd9c1ca..e32b5524f7f 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -909,6 +909,12 @@ const CONFIG = { get PROXIES () { return config.get('import.videos.http.proxies') + }, + + COOKIES: { + get ENABLED () { + return config.get('import.videos.http.cookies.enabled') + } } }, TORRENT: { diff --git a/support/docker/production/.env b/support/docker/production/.env index c1b872ee481..208280ead89 100644 --- a/support/docker/production/.env +++ b/support/docker/production/.env @@ -102,6 +102,34 @@ PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PUBLIC="public-read" PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PRIVATE="private" +# ----------------------------------------------------------------------------- +# Outbound Proxies (Optional) +# ----------------------------------------------------------------------------- +# These proxies affect all outbound PeerTube HTTP(S) traffic, including yt-dlp. +# This is different from the reverse proxy in front of PeerTube. +#HTTP_PROXY=http://username:password@proxy.example:3128 +#HTTPS_PROXY=http://username:password@proxy.example:3128 + + +# ----------------------------------------------------------------------------- +# HTTP Video Imports (Optional) +# ----------------------------------------------------------------------------- +#PEERTUBE_IMPORT_VIDEOS_HTTP=true +#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_URL=https://api.github.com/repos/yt-dlp/yt-dlp/releases +#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_NAME=yt-dlp +#PEERTUBE_IMPORT_VIDEOS_HTTP_FORCE_IPV4=true +# Dedicated outbound proxy pool for HTTP imports. PeerTube randomly selects one proxy from this list. +# JSON array. Example: ["http://user:pass@proxy-1:3128","http://user:pass@proxy-2:3128"] +#PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES=[] +# Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited. +# Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported. +# That account may be rate limited or temporarily blocked. +# Put your Netscape-format cookies in /data/tmp-persistent/youtube-cookies.txt. +# See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions. +# See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup. +#PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED=false + + # ----------------------------------------------------------------------------- # Logging (Optional) # ----------------------------------------------------------------------------- diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml index 4155fd4f9cd..098d4fe9140 100644 --- a/support/docker/production/config/custom-environment-variables.yaml +++ b/support/docker/production/config/custom-environment-variables.yaml @@ -778,6 +778,10 @@ import: proxies: __name: PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES __format: json + cookies: + enabled: + __name: PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED + __format: json torrent: enabled: __name: PEERTUBE_IMPORT_VIDEOS_TORRENT