diff --git a/api/package.json b/api/package.json index 664ffbd72..a5fcfb58b 100644 --- a/api/package.json +++ b/api/package.json @@ -39,7 +39,7 @@ "set-cookie-parser": "2.6.0", "undici": "^6.21.3", "url-pattern": "1.0.3", - "youtubei.js": "15.1.1", + "youtubei.js": "16.0.0", "zod": "^3.23.8" }, "optionalDependencies": { diff --git a/api/src/processing/helpers/youtube-session.js b/api/src/processing/helpers/youtube-session.js index 85f1a6e13..d4b5dfd04 100644 --- a/api/src/processing/helpers/youtube-session.js +++ b/api/src/processing/helpers/youtube-session.js @@ -9,6 +9,10 @@ const defaultAgent = new Agent(); let session; const validateSession = (sessionResponse) => { + sessionResponse.visitor_data ??= sessionResponse.contentBinding; + sessionResponse.potoken ??= sessionResponse.poToken; + sessionResponse.updated ??= new Date().getTime(); + if (!sessionResponse.potoken) { throw "no poToken in session response"; } @@ -33,11 +37,11 @@ const updateSession = (newSession) => { const loadSession = async () => { const sessionServerUrl = new URL(env.ytSessionServer); - sessionServerUrl.pathname = "/token"; + sessionServerUrl.pathname = "/get_pot"; const newSession = await fetch( sessionServerUrl, - { dispatcher: defaultAgent } + { method: 'POST', dispatcher: defaultAgent } ).then(a => a.json()); validateSession(newSession); diff --git a/api/src/processing/services/tumblr.js b/api/src/processing/services/tumblr.js index 2b8aa4ce2..22e58fdc1 100644 --- a/api/src/processing/services/tumblr.js +++ b/api/src/processing/services/tumblr.js @@ -1,61 +1,106 @@ import psl from "@imput/psl"; +const LEGACY_API = 'https://api.tumblr.com'; +const NEW_API = 'https://api-http2.tumblr.com'; const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z'; -const API_BASE = 'https://api-http2.tumblr.com'; -function request(domain, id) { - const url = new URL(`/v2/blog/${domain}/posts/${id}/permalink`, API_BASE); +async function fetchLegacy(domain, id, type) { + const url = new URL(`/v2/blog/${domain}/posts`, LEGACY_API); + url.searchParams.set('id', id); + if (type) url.searchParams.set('type', type); url.searchParams.set('api_key', API_KEY); - url.searchParams.set('fields[blogs]', 'uuid,name,avatar,?description,?can_message,?can_be_followed,?is_adult,?reply_conditions,' - + '?theme,?title,?url,?is_blocked_from_primary,?placement_id,?primary,?updated,?followed,' - + '?ask,?can_subscribe,?paywall_access,?subscription_plan,?is_blogless_advertiser,?tumblrmart_accessories'); - - return fetch(url, { - headers: { - 'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr', - 'X-Version': 'iPhone/33.3/333010/17.3.1/tumblr' - } - }).then(a => a.json()).catch(() => {}); + + try { + const res = await fetch(url.toString(), { + headers: { 'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr' } + }); + return await res.json(); + } catch { + return null; + } +} + +async function fetchNew(domain, id) { + const url = new URL(`/v2/blog/${domain}/posts/${id}`, NEW_API); + url.searchParams.set('api_key', API_KEY); + url.searchParams.set('fields[blogs]', 'uuid,name,avatar,?description,?can_message,?can_be_followed,?is_adult,?reply_conditions,' + + '?theme,?title,?url,?is_blocked_from_primary,?placement_id,?primary,?updated,?followed,' + + '?ask,?can_subscribe,?paywall_access,?subscription_plan,?is_blogless_advertiser,?tumblrmart_accessories'); + + try { + const res = await fetch(url.toString(), { + headers: { + 'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr', + 'X-Version': 'iPhone/33.3/333010/17.3.1/tumblr' + } + }); + return await res.json(); + } catch { + return null; + } } export default async function(input) { let { subdomain } = psl.parse(input.url.hostname); - if (subdomain?.includes('.')) { - return { error: "link.unsupported" }; - } else if (subdomain === 'www' || subdomain === 'at') { - subdomain = undefined - } + if (subdomain?.includes('.')) return { error: "link.unsupported" }; + else if (subdomain === 'www' || subdomain === 'at') subdomain = undefined; const domain = `${subdomain ?? input.user}.tumblr.com`; - const data = await request(domain, input.id); + let data = await fetchLegacy(domain, input.id, 'audio'); + let post = data?.response?.posts?.[0]; + + if (post && post.type === 'audio') { + const audioUrl = post.audio_url || post.audio_source_url; + const title = post.caption ? post.caption.replace(/<[^>]+>/g, '').slice(0, 100) : ''; + return { + urls: audioUrl, + filenameAttributes: { + service: 'tumblr', + id: input.id, + title, + author: post.blog_name + }, + isAudioOnly: true, + bestAudio: "mp3" + }; + } + + data = await fetchLegacy(domain, input.id, 'video'); + post = data?.response?.posts?.[0]; + + if (post && post.type === 'video') { + const videoUrl = post.video_url || post.media?.find(m => m.type === 'video')?.url; + return { + urls: videoUrl, + filename: `tumblr_${input.id}.mp4`, + audioFilename: `tumblr_${input.id}_audio` + }; + } + + data = await fetchNew(domain, input.id); const element = data?.response?.timeline?.elements?.[0]; if (!element) return { error: "fetch.empty" }; const contents = [ ...element.content, ...element?.trail?.map(t => t.content).flat() - ] + ]; const audio = contents.find(c => c.type === 'audio'); if (audio && audio.provider === 'tumblr') { - const fileMetadata = { - title: audio?.title, - artist: audio?.artist - }; - return { urls: audio.media.url, filenameAttributes: { service: 'tumblr', id: input.id, - title: fileMetadata.title, - author: fileMetadata.artist + title: audio?.title, + author: audio?.artist }, isAudioOnly: true, bestAudio: "mp3", - } + }; } const video = contents.find(c => c.type === 'video'); @@ -64,8 +109,8 @@ export default async function(input) { urls: video.media.url, filename: `tumblr_${input.id}.mp4`, audioFilename: `tumblr_${input.id}_audio` - } + }; } - return { error: "link.unsupported" } + return { error: "link.unsupported" }; } diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index b47c266d3..c1df46a67 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -1,12 +1,29 @@ import HLS from "hls-parser"; -import { fetch } from "undici"; -import { Innertube, Session } from "youtubei.js"; +import { fetch, Request } from "undici"; +import { Innertube, Platform, Session } from "youtubei.js"; import { env } from "../../config.js"; import { getCookie } from "../cookie/manager.js"; import { getYouTubeSession } from "../helpers/youtube-session.js"; +// https://github.com/LuanRT/YouTube.js/pull/1052 +Platform.shim.eval = async (data, env) => { + const properties = []; + + if (env.n) { + properties.push(`n: exportedVars.nFunction("${env.n}")`) + } + + if (env.sig) { + properties.push(`sig: exportedVars.sigFunction("${env.sig}")`) + } + + const code = `${data.output}\nreturn { ${properties.join(', ')} }`; + + return new Function(code)(); +} + const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms let innertube, lastRefreshedAt; @@ -206,10 +223,24 @@ export default async function (o) { let yt; try { yt = await cloneInnertube( - (input, init) => fetch(input, { - ...init, - dispatcher: o.dispatcher - }), + (input, init) => { + const url = typeof input === 'string' + ? new URL(input) + : input instanceof URL + ? input + : new URL(input.url); + + const request = new Request( + url, + input instanceof Platform.shim.Request + ? input : undefined + ); + + return fetch(request, { + ...init, + dispatcher: o.dispatcher + }); + }, useSession ); } catch (e) { @@ -529,7 +560,7 @@ export default async function (o) { } if (!clientsWithNoCipher.includes(innertubeClient) && innertube) { - urls = audio.decipher(innertube.session.player); + urls = await audio.decipher(innertube.session.player); } let cover = `https://i.ytimg.com/vi/${o.id}/maxresdefault.jpg`; @@ -576,8 +607,8 @@ export default async function (o) { filenameAttributes.extension = o.container === "auto" ? codecList[codec].container : o.container; if (!clientsWithNoCipher.includes(innertubeClient) && innertube) { - video = video.decipher(innertube.session.player); - audio = audio.decipher(innertube.session.player); + video = await video.decipher(innertube.session.player); + audio = await audio.decipher(innertube.session.player); } else { video = video.url; audio = audio.url; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 99bc803c4..daefd140b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,8 +59,8 @@ importers: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: 15.1.1 - version: 15.1.1 + specifier: 16.0.0 + version: 16.0.0 zod: specifier: ^3.23.8 version: 3.23.8 @@ -1447,9 +1447,6 @@ packages: resolution: {integrity: sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==} engines: {node: 20 || >=22} - jintr@3.3.1: - resolution: {integrity: sha512-nnOzyhf0SLpbWuZ270Omwbj5LcXUkTcZkVnK8/veJXtSZOiATM5gMZMdmzN75FmTyj+NVgrGaPdH12zIJ24oIA==} - joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -1538,6 +1535,10 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + meriyah@6.1.4: + resolution: {integrity: sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==} + engines: {node: '>=18.0.0'} + methods@1.1.2: resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} engines: {node: '>= 0.6'} @@ -2186,8 +2187,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@15.1.1: - resolution: {integrity: sha512-fuEDj9Ky6cAQg93BrRVCbr+GTYNZQAIFZrx/a3oDRuGc3Mf5bS0dQfoYwwgjtSV7sgAKQEEdGtzRdBzOc8g72Q==} + youtubei.js@16.0.0: + resolution: {integrity: sha512-aMx+ulnrxzsgVsxTr7gbBVnIjti2NQUlMwCoo1/MzICCJS3iMLOPUFdo7bSpwskL6ljzQ/LxmmB4WQC3FtkBlA==} zimmerframe@1.1.2: resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==} @@ -3388,10 +3389,6 @@ snapshots: dependencies: '@isaacs/cliui': 8.0.2 - jintr@3.3.1: - dependencies: - acorn: 8.14.0 - joycon@3.1.1: {} js-yaml@3.14.1: @@ -3464,6 +3461,8 @@ snapshots: merge2@1.4.1: {} + meriyah@6.1.4: {} + methods@1.1.2: {} micromatch@4.0.7: @@ -4044,11 +4043,10 @@ snapshots: yocto-queue@0.1.0: {} - youtubei.js@15.1.1: + youtubei.js@16.0.0: dependencies: '@bufbuild/protobuf': 2.2.5 - jintr: 3.3.1 - undici: 6.21.3 + meriyah: 6.1.4 zimmerframe@1.1.2: {}