Add cookies to yt-dlp config and docker env

config/default.yaml

@@ -884,6 +884,15 @@ import:
       proxies:
 #        - "https://username:password@example.com:8888"
 
+      cookies:
+        # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited
+        # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported
+        # That account may be rate limited or temporarily blocked
+        # PeerTube expects a Netscape-format file at storage/tmp-persistent/youtube-cookies.txt
+        # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions
+        # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup
+        enabled: false
+
     # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
     torrent:
       # We recommend to only enable magnet URI/torrent import if you trust your users

config/production.yaml.example

@@ -894,6 +894,15 @@ import:
       proxies:
 #        - "https://username:password@example.com:8888"
 
+      cookies:
+        # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited
+        # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported
+        # That account may be rate limited or temporarily blocked
+        # PeerTube expects a Netscape-format file at /var/www/peertube/storage/tmp-persistent/youtube-cookies.txt
+        # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions
+        # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup
+        enabled: false
+
     # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
     torrent:
       # We recommend to only enable magnet URI/torrent import if you trust your users

packages/tests/src/server-helpers/youtube-dl.ts

@@ -1,7 +1,11 @@
 /* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import { expect } from 'chai'
+import { mkdtemp, rm, writeFile } from 'fs/promises'
+import { tmpdir } from 'os'
+import { join } from 'path'
 import { YoutubeDLCLI } from '@peertube/peertube-server/core/helpers/youtube-dl/youtube-dl-cli.js'
+import { logger } from '@peertube/peertube-server/core/helpers/logger.js'
 import { CONFIG } from '@peertube/peertube-server/core/initializers/config.js'
 
 describe('YoutubeDLCLI', function () {
@@ -91,4 +95,96 @@ describe('YoutubeDLCLI', function () {
       }
     })
   })
+
+  describe('wrapWithCookiesOptions', function () {
+    let cli: any
+
+    before(function () {
+      cli = Object.create(YoutubeDLCLI.prototype)
+    })
+
+    it('Should prepend cookies file when configured and the file exists', async function () {
+      const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR')
+      const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')
+      const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-'))
+      const cookiesFile = join(tempDir, 'youtube-cookies.txt')
+
+      await writeFile(cookiesFile, '# Netscape HTTP Cookie File\n')
+
+      Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', {
+        get: () => tempDir,
+        configurable: true
+      })
+
+      Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {
+        get: () => true,
+        configurable: true
+      })
+
+      try {
+        const inputArgs = [ '--dump-json', '-f', 'best' ]
+        const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)
+
+        expect(result).to.deep.equal([ '--cookies', cookiesFile, ...inputArgs ])
+      } finally {
+        Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor)
+        Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)
+        await rm(tempDir, { recursive: true, force: true })
+      }
+    })
+
+    it('Should log an error and continue when the cookies file is missing', async function () {
+      const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR')
+      const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')
+      const originalLoggerError = logger.error
+      const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-'))
+      const loggedMessages: any[][] = []
+
+      Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', {
+        get: () => tempDir,
+        configurable: true
+      })
+
+      Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {
+        get: () => true,
+        configurable: true
+      })
+
+      ;(logger as any).error = (...args: any[]) => {
+        loggedMessages.push(args)
+      }
+
+      try {
+        const inputArgs = [ '--dump-json', '-f', 'best' ]
+        const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)
+
+        expect(result).to.deep.equal(inputArgs)
+        expect(loggedMessages).to.have.lengthOf(1)
+        expect(loggedMessages[0][0]).to.contain('yt-dlp cookies are enabled but the cookies file %s does not exist')
+      } finally {
+        Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor)
+        Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)
+        ;(logger as any).error = originalLoggerError
+        await rm(tempDir, { recursive: true, force: true })
+      }
+    })
+
+    it('Should not modify args when cookies are disabled', async function () {
+      const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')
+
+      Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {
+        get: () => false,
+        configurable: true
+      })
+
+      try {
+        const inputArgs = [ '--dump-json', '-f', 'best' ]
+        const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)
+
+        expect(result).to.deep.equal(inputArgs)
+      } finally {
+        Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)
+      }
+    })
+  })
 })

server/core/helpers/youtube-dl/youtube-dl-cli.ts

@@ -215,6 +215,7 @@ export class YoutubeDLCLI {
 
     let completeArgs = this.wrapWithJSRuntimeOptions(args)
     completeArgs = this.wrapWithProxyOptions(completeArgs)
+    completeArgs = await this.wrapWithCookiesOptions(completeArgs)
     completeArgs = this.wrapWithIPOptions(completeArgs)
     completeArgs = this.wrapWithFFmpegOptions(completeArgs)
 
@@ -272,6 +273,28 @@ export class YoutubeDLCLI {
     return args
   }
 
+  private async wrapWithCookiesOptions (args: string[]) {
+    if (!CONFIG.IMPORT.VIDEOS.HTTP.COOKIES.ENABLED) {
+      return args
+    }
+
+    const cookiesPath = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, 'youtube-cookies.txt')
+
+    if (!await pathExists(cookiesPath)) {
+      logger.error(
+        'yt-dlp cookies are enabled but the cookies file %s does not exist. Continuing without cookies.',
+        cookiesPath,
+        lTags()
+      )
+
+      return args
+    }
+
+    logger.debug('Using cookies file %s for YoutubeDL', cookiesPath, lTags())
+
+    return [ '--cookies', cookiesPath ].concat(args)
+  }
+
   private wrapWithFFmpegOptions (args: string[]) {
     if (process.env.FFMPEG_PATH) {
       logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())

server/core/initializers/checker-before-init.ts

@@ -119,6 +119,7 @@ export function checkMissedConfig () {
     'import.videos.timeout',
     'import.videos.http.force_ipv4',
     'import.videos.http.proxies',
+    'import.videos.http.cookies.enabled',
     'import.video_channel_synchronization.enabled',
     'import.video_channel_synchronization.max_per_user',
     'import.video_channel_synchronization.check_interval',

server/core/initializers/config.ts

@@ -909,6 +909,12 @@ const CONFIG = {
 
         get PROXIES () {
           return config.get<string[]>('import.videos.http.proxies')
+        },
+
+        COOKIES: {
+          get ENABLED () {
+            return config.get<boolean>('import.videos.http.cookies.enabled')
+          }
         }
       },
       TORRENT: {

support/docker/production/.env

@@ -102,6 +102,34 @@ PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PUBLIC="public-read"
 PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PRIVATE="private"
 
 
+# -----------------------------------------------------------------------------
+# Outbound Proxies (Optional)
+# -----------------------------------------------------------------------------
+# These proxies affect all outbound PeerTube HTTP(S) traffic, including yt-dlp.
+# This is different from the reverse proxy in front of PeerTube.
+#HTTP_PROXY=http://username:password@proxy.example:3128
+#HTTPS_PROXY=http://username:password@proxy.example:3128
+
+
+# -----------------------------------------------------------------------------
+# HTTP Video Imports (Optional)
+# -----------------------------------------------------------------------------
+#PEERTUBE_IMPORT_VIDEOS_HTTP=true
+#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_URL=https://api.github.com/repos/yt-dlp/yt-dlp/releases
+#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_NAME=yt-dlp
+#PEERTUBE_IMPORT_VIDEOS_HTTP_FORCE_IPV4=true
+# Dedicated outbound proxy pool for HTTP imports. PeerTube randomly selects one proxy from this list.
+# JSON array. Example: ["http://user:pass@proxy-1:3128","http://user:pass@proxy-2:3128"]
+#PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES=[]
+# Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited.
+# Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported.
+# That account may be rate limited or temporarily blocked.
+# Put your Netscape-format cookies in /data/tmp-persistent/youtube-cookies.txt.
+# See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions.
+# See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup.
+#PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED=false
+
+
 # -----------------------------------------------------------------------------
 # Logging (Optional)
 # -----------------------------------------------------------------------------

support/docker/production/config/custom-environment-variables.yaml

@@ -778,6 +778,10 @@ import:
       proxies:
         __name: PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES
         __format: json
+      cookies:
+        enabled:
+          __name: PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED
+          __format: json
     torrent:
       enabled:
         __name: PEERTUBE_IMPORT_VIDEOS_TORRENT