From 4b50ee1073e351ce0d0380c2fb245d6fbf960d99 Mon Sep 17 00:00:00 2001 From: Damir Modyarov Date: Sat, 11 Jan 2025 00:21:50 +0300 Subject: [PATCH 1/3] feat: Implement custom instance support --- locales/en.ftl | 4 + locales/ru.ftl | 4 + migrations/0001_eminent_ultron.sql | 1 + migrations/meta/0001_snapshot.json | 129 +++++++++++++++++++++++++++++ migrations/meta/_journal.json | 7 ++ package.json | 1 + pnpm-lock.yaml | 9 ++ src/core/data/db/schema.ts | 1 + src/core/data/request.ts | 29 +++++++ src/core/data/settings.ts | 3 + src/core/utils/url.ts | 20 +++++ src/telegram/helpers/handler.ts | 8 +- src/telegram/helpers/i18n.ts | 10 +-- 13 files changed, 219 insertions(+), 7 deletions(-) create mode 100644 migrations/0001_eminent_ultron.sql create mode 100644 migrations/meta/0001_snapshot.json create mode 100644 src/core/utils/url.ts diff --git a/locales/en.ftl b/locales/en.ftl index 29a5a38..2653061 100644 --- a/locales/en.ftl +++ b/locales/en.ftl @@ -42,5 +42,9 @@ setting-lang-ru = русский setting-lang-uk-UA = українська setting-lang-unset = same as telegram +setting-instance = custom instance +setting-instance-unset = disabled +setting-instance-custom = override + stats-personal = i helped you with downloading { $count } times! (˶ᵔ ᵕ ᵔ˶) stats-global = i helped with downloading { $count } times! (˶ᵔ ᵕ ᵔ˶) \ No newline at end of file diff --git a/locales/ru.ftl b/locales/ru.ftl index fdf29b7..fcabfd3 100644 --- a/locales/ru.ftl +++ b/locales/ru.ftl @@ -37,5 +37,9 @@ setting-attribution-1 = давай setting-lang = язык setting-lang-unset = как в тг +setting-instance = кастомный инстанс +setting-instance-unset = выключен +setting-instance-custom = настроить + stats-personal = я помог тебе с загрузкой { $count } раз! (˶ᵔ ᵕ ᵔ˶) stats-global = я помог с загрузкой { $count } раз! (˶ᵔ ᵕ ᵔ˶) \ No newline at end of file diff --git a/migrations/0001_eminent_ultron.sql b/migrations/0001_eminent_ultron.sql new file mode 100644 index 0000000..ba83a0b --- /dev/null +++ b/migrations/0001_eminent_ultron.sql @@ -0,0 +1 @@ +ALTER TABLE settings ADD `instance` text; \ No newline at end of file diff --git a/migrations/meta/0001_snapshot.json b/migrations/meta/0001_snapshot.json new file mode 100644 index 0000000..3df4372 --- /dev/null +++ b/migrations/meta/0001_snapshot.json @@ -0,0 +1,129 @@ +{ + "version": "5", + "dialect": "sqlite", + "id": "8f524a1e-29ad-4518-88f2-435c6a3114f0", + "prevId": "a3959bd4-7853-4642-b401-7ed157ba3283", + "tables": { + "requests": { + "name": "requests", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "author_id": { + "name": "author_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attribution": { + "name": "attribution", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "instance": { + "name": "instance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "settings_id_unique": { + "name": "settings_id_unique", + "columns": [ + "id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "downloads": { + "name": "downloads", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "users_id_unique": { + "name": "users_id_unique", + "columns": [ + "id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/migrations/meta/_journal.json b/migrations/meta/_journal.json index b885043..8d10b45 100644 --- a/migrations/meta/_journal.json +++ b/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1720203773408, "tag": "0000_supreme_ben_urich", "breakpoints": true + }, + { + "idx": 1, + "version": "5", + "when": 1736519894021, + "tag": "0001_eminent_ultron", + "breakpoints": true } ] } \ No newline at end of file diff --git a/package.json b/package.json index cece120..76ecbe4 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "better-sqlite3": "^11.5.0", "dotenv": "^16.3.1", "drizzle-orm": "^0.29.3", + "ipaddr.js": "^2.2.0", "mediainfo.js": "^0.3.2", "zod": "^3.22.4" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2072e10..2c3dcf8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,9 @@ importers: drizzle-orm: specifier: ^0.29.3 version: 0.29.3(@types/better-sqlite3@7.6.11)(better-sqlite3@11.5.0) + ipaddr.js: + specifier: ^2.2.0 + version: 2.2.0 mediainfo.js: specifier: ^0.3.2 version: 0.3.2 @@ -1794,6 +1797,10 @@ packages: ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + ipaddr.js@2.2.0: + resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + engines: {node: '>= 10'} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} @@ -4353,6 +4360,8 @@ snapshots: ini@1.3.8: {} + ipaddr.js@2.2.0: {} + is-arrayish@0.2.1: {} is-builtin-module@3.2.1: diff --git a/src/core/data/db/schema.ts b/src/core/data/db/schema.ts index d25650f..2987317 100644 --- a/src/core/data/db/schema.ts +++ b/src/core/data/db/schema.ts @@ -16,4 +16,5 @@ export const settings = sqliteTable("settings", { preferredOutput: text("output"), preferredAttribution: int("attribution").notNull().default(0), languageOverride: text("language"), + instanceOverride: text("instance"), }) diff --git a/src/core/data/request.ts b/src/core/data/request.ts index b3a0e9c..40fd2ed 100644 --- a/src/core/data/request.ts +++ b/src/core/data/request.ts @@ -12,18 +12,21 @@ import type { Result } from "@/core/utils/result" import { error, ok } from "@/core/utils/result" import type { CompoundText, Text } from "@/core/utils/text" import { compound, literal, translatable } from "@/core/utils/text" +import { safeUrlSchema } from "@/core/utils/url" export const apiServerSchema = z.object({ name: z.string().optional(), url: z.string().url(), auth: z.string().optional(), youtubeHls: z.boolean().optional(), + unsafe: z.boolean().optional(), }).or( z.string().url().transform(data => ({ name: undefined, url: data, auth: undefined, youtubeHls: undefined, + unsafe: undefined, })), ).transform(data => ({ ...data, @@ -134,6 +137,16 @@ async function tryDownload(outputType: string, request: MediaRequest, apiPool: A if (!currentApi) return error(compound(...fails)) + if (currentApi.unsafe && !(await safeUrlSchema.safeParseAsync(currentApi.url)).success) { + return tryDownload( + outputType, + request, + apiPool.slice(1), + lang, + [...fails, compound(literal(`\n${currentApi.name}: `), translatable("error-invalid-custom-instance"))], + ) + } + const res = await fetchMedia({ url: request.url, downloadMode: outputType, @@ -153,5 +166,21 @@ async function tryDownload(outputType: string, request: MediaRequest, apiPool: A ) } + if (currentApi.unsafe) { + if ( + (res.result.status === "picker" && !(await safeUrlSchema.safeParseAsync(res.result.audio)).success) + || (res.result.status === "tunnel" && !(await safeUrlSchema.safeParseAsync(res.result.url)).success) + || (res.result.status === "redirect" && !(await safeUrlSchema.safeParseAsync(res.result.url)).success) + ) { + return tryDownload( + outputType, + request, + apiPool.slice(1), + lang, + [...fails, literal(`\n${currentApi.name}: unsafe api response`)], + ) + } + } + return res } diff --git a/src/core/data/settings.ts b/src/core/data/settings.ts index 4fe001b..fcf60e3 100644 --- a/src/core/data/settings.ts +++ b/src/core/data/settings.ts @@ -13,6 +13,7 @@ export const defaultSettings: Settings = { preferredOutput: null, preferredAttribution: 0, languageOverride: null, + instanceOverride: null, } export const settingOptions: { @@ -21,6 +22,7 @@ export const settingOptions: { preferredOutput: [null, ...outputOptions], preferredAttribution: [0, 1], languageOverride: [null, ...locales], + instanceOverride: [null, customValue], } export const settingI18n: { @@ -29,6 +31,7 @@ export const settingI18n: { preferredOutput: { key: "output", mode: "translatable" }, preferredAttribution: { key: "attribution", mode: "translatable" }, languageOverride: { key: "lang", mode: "translatable" }, + instanceOverride: { key: "instance", mode: "literal" }, } export async function getSettings(id: number): Promise { diff --git a/src/core/utils/url.ts b/src/core/utils/url.ts new file mode 100644 index 0000000..4ca02ee --- /dev/null +++ b/src/core/utils/url.ts @@ -0,0 +1,20 @@ +import * as dns from "node:dns/promises" + +import ipaddr from "ipaddr.js" +import { z } from "zod" + +export const safeUrlSchema = z + .string() + .url() + .refine(async (u) => { + if (!URL.canParse(u)) + return false + const url = new URL(u) + if (ipaddr.isValid(url.hostname) && ipaddr.parse(url.hostname).range() !== "unicast") + return false + const res = await dns.lookup(url.hostname, { all: true }).catch(() => null) + if (!res || !res.every(i => ipaddr.parse(i.address).range() === "unicast")) + return false + return url.protocol === "https:" + }) + .nullable() diff --git a/src/telegram/helpers/handler.ts b/src/telegram/helpers/handler.ts index 46e5868..9a2e396 100644 --- a/src/telegram/helpers/handler.ts +++ b/src/telegram/helpers/handler.ts @@ -4,7 +4,7 @@ import type { GeneralTrack, ImageTrack, VideoTrack } from "mediainfo.js" import { CallbackDataBuilder } from "@mtcute/dispatcher" import mediaInfoFactory from "mediainfo.js" -import type { MediaRequest } from "@/core/data/request" +import type { ApiServer, MediaRequest } from "@/core/data/request" import { finishRequest, outputOptions } from "@/core/data/request" import type { Result } from "@/core/utils/result" import { error, ok } from "@/core/utils/result" @@ -12,6 +12,7 @@ import type { Text } from "@/core/utils/text" import { translatable } from "@/core/utils/text" import { env } from "@/telegram/helpers/env" import { getPeerLocale } from "@/telegram/helpers/i18n" +import { getPeerSettings } from "@/telegram/helpers/settings" export const OutputButton = new CallbackDataBuilder("dl", "output", "request") export const getOutputSelectionMessage = (requestId: string) => ({ @@ -86,7 +87,10 @@ async function fileToInputMedia(file: ArrayBuffer, fileName?: string): Promise> { if (!request) return error(translatable("error-request-not-found")) - const res = await finishRequest(outputType, request, env.API_ENDPOINTS, await getPeerLocale(peer)) + const settings = await getPeerSettings(peer) + const locale = settings.languageOverride ?? getPeerLocale(peer) + const endpoints: ApiServer[] = settings.instanceOverride ? [{ name: "custom", url: settings.instanceOverride, unsafe: true }] : env.API_ENDPOINTS + const res = await finishRequest(outputType, request, endpoints, locale) if (!res.success) return res diff --git a/src/telegram/helpers/i18n.ts b/src/telegram/helpers/i18n.ts index 54ec000..053cd2c 100644 --- a/src/telegram/helpers/i18n.ts +++ b/src/telegram/helpers/i18n.ts @@ -4,13 +4,13 @@ import type { TranslationParams } from "@/core/utils/i18n" import { fallbackLocale, translate } from "@/core/utils/i18n" import { getPeerSettings } from "@/telegram/helpers/settings" -export async function getPeerLocale(peer: Peer) { - const { languageOverride } = await getPeerSettings(peer) - return languageOverride ?? ("language" in peer ? peer.language : null) ?? fallbackLocale +export function getPeerLocale(peer: Peer) { + return ("language" in peer ? peer.language : null) ?? fallbackLocale } export type Translator = Awaited> export async function translatorFor(peer: Peer) { - const locale = await getPeerLocale(peer) - return (key: string, params?: TranslationParams) => translate(locale, key, params) + const { languageOverride } = await getPeerSettings(peer) + const locale = getPeerLocale(peer) + return (key: string, params?: TranslationParams) => translate(languageOverride ?? locale, key, params) } From 71e98fcef9d4977ce1c43ff60c14fa9a4db7a095 Mon Sep 17 00:00:00 2001 From: Damir Modyarov Date: Sat, 8 Feb 2025 18:53:00 +0300 Subject: [PATCH 2/3] feat: Implement support for instance urls with auth --- src/core/data/request.ts | 8 ++++---- src/core/utils/url.ts | 13 +++++++++++++ src/telegram/helpers/handler.ts | 5 ++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/core/data/request.ts b/src/core/data/request.ts index 40fd2ed..807e0c5 100644 --- a/src/core/data/request.ts +++ b/src/core/data/request.ts @@ -12,7 +12,7 @@ import type { Result } from "@/core/utils/result" import { error, ok } from "@/core/utils/result" import type { CompoundText, Text } from "@/core/utils/text" import { compound, literal, translatable } from "@/core/utils/text" -import { safeUrlSchema } from "@/core/utils/url" +import { safeUrlSchema, urlWithAuthSchema } from "@/core/utils/url" export const apiServerSchema = z.object({ name: z.string().optional(), @@ -21,10 +21,10 @@ export const apiServerSchema = z.object({ youtubeHls: z.boolean().optional(), unsafe: z.boolean().optional(), }).or( - z.string().url().transform(data => ({ + urlWithAuthSchema.transform(data => ({ name: undefined, - url: data, - auth: undefined, + url: data.url, + auth: data.auth, youtubeHls: undefined, unsafe: undefined, })), diff --git a/src/core/utils/url.ts b/src/core/utils/url.ts index 4ca02ee..0d738bc 100644 --- a/src/core/utils/url.ts +++ b/src/core/utils/url.ts @@ -18,3 +18,16 @@ export const safeUrlSchema = z return url.protocol === "https:" }) .nullable() + +export const urlWithAuthSchema = z + .string() + .url() + .transform((u) => { + const url = new URL(u) + if (!url.username && !url.password) + return { url: u } + const auth = url.password ? `${url.username} ${url.password}` : url.username + url.username = "" + url.password = "" + return { url: url.href, auth } + }) diff --git a/src/telegram/helpers/handler.ts b/src/telegram/helpers/handler.ts index 9a2e396..7733739 100644 --- a/src/telegram/helpers/handler.ts +++ b/src/telegram/helpers/handler.ts @@ -10,6 +10,7 @@ import type { Result } from "@/core/utils/result" import { error, ok } from "@/core/utils/result" import type { Text } from "@/core/utils/text" import { translatable } from "@/core/utils/text" +import { urlWithAuthSchema } from "@/core/utils/url" import { env } from "@/telegram/helpers/env" import { getPeerLocale } from "@/telegram/helpers/i18n" import { getPeerSettings } from "@/telegram/helpers/settings" @@ -89,7 +90,9 @@ export async function handleMediaDownload(outputType: string, request: MediaRequ return error(translatable("error-request-not-found")) const settings = await getPeerSettings(peer) const locale = settings.languageOverride ?? getPeerLocale(peer) - const endpoints: ApiServer[] = settings.instanceOverride ? [{ name: "custom", url: settings.instanceOverride, unsafe: true }] : env.API_ENDPOINTS + const endpoints: ApiServer[] = settings.instanceOverride + ? [{ name: "custom", ...urlWithAuthSchema.parse(settings.instanceOverride), unsafe: true }] + : env.API_ENDPOINTS const res = await finishRequest(outputType, request, endpoints, locale) if (!res.success) return res From 5cb4d20e3eca6505fa80b41390f73184bbde9de0 Mon Sep 17 00:00:00 2001 From: Damir Modyarov Date: Tue, 11 Mar 2025 22:49:37 +0300 Subject: [PATCH 3/3] feat: Rewrite cobalt data layer with @fuman/fetch and implement proxy support --- .env.example | 3 +- package.json | 7 +- pnpm-lock.yaml | 119 +++++++++++++++++++------- src/core/data/cobalt.ts | 142 ------------------------------- src/core/data/cobalt/common.ts | 15 ++++ src/core/data/cobalt/download.ts | 95 +++++++++++++++++++++ src/core/data/cobalt/error.ts | 27 ++++++ src/core/data/cobalt/external.ts | 5 ++ src/core/data/cobalt/index.ts | 5 ++ src/core/data/cobalt/server.ts | 25 ++++++ src/core/data/cobalt/tunnel.ts | 24 ++++++ src/core/data/request.ts | 47 +++------- src/core/utils/compose.ts | 8 ++ src/core/utils/proxy.ts | 15 ++++ src/telegram/helpers/env.ts | 3 +- src/telegram/helpers/handler.ts | 5 +- 16 files changed, 335 insertions(+), 210 deletions(-) delete mode 100644 src/core/data/cobalt.ts create mode 100644 src/core/data/cobalt/common.ts create mode 100644 src/core/data/cobalt/download.ts create mode 100644 src/core/data/cobalt/error.ts create mode 100644 src/core/data/cobalt/external.ts create mode 100644 src/core/data/cobalt/index.ts create mode 100644 src/core/data/cobalt/server.ts create mode 100644 src/core/data/cobalt/tunnel.ts create mode 100644 src/core/utils/compose.ts create mode 100644 src/core/utils/proxy.ts diff --git a/.env.example b/.env.example index 81855b0..41e6e4c 100644 --- a/.env.example +++ b/.env.example @@ -5,9 +5,10 @@ BOT_TOKEN= API_ENDPOINTS='[ "https://api.cobalt.tools", { "url": "https://api.cobalt.tools" }, - { "name": "custom name", "url": "https://api.cobalt.tools", "auth": "Api-Key 123" } + { "name": "custom name", "url": "https://api.cobalt.tools", "auth": "Api-Key 123", "proxy": "http://proxy.example.com" } ]' # Optional variables, defaults shown SELECT_TYPE_PHOTO_URL=https://i.otomir23.me/buckets/cobold/download.png ERROR_CHAT_ID= +CUSTOM_INSTANCE_PROXY_URL= diff --git a/package.json b/package.json index 76ecbe4..c8818aa 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "dependencies": { "@fluent/bundle": "^0.18.0", "@fluent/langneg": "^0.7.0", + "@fuman/fetch": "^0.0.12", "@mtcute/crypto-node": "^0.19.0", "@mtcute/dispatcher": "^0.19.1", "@mtcute/node": "^0.19.3", @@ -46,9 +47,13 @@ "drizzle-orm": "^0.29.3", "ipaddr.js": "^2.2.0", "mediainfo.js": "^0.3.2", + "undici": "^7.4.0", "zod": "^3.22.4" }, "pnpm": { - "onlyBuiltDependencies": ["better-sqlite3", "esbuild"] + "onlyBuiltDependencies": [ + "better-sqlite3", + "esbuild" + ] } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c3dcf8..31b351e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@fluent/langneg': specifier: ^0.7.0 version: 0.7.0 + '@fuman/fetch': + specifier: ^0.0.12 + version: 0.0.12(zod@3.22.4) '@mtcute/crypto-node': specifier: ^0.19.0 version: 0.19.0 @@ -41,6 +44,9 @@ importers: mediainfo.js: specifier: ^0.3.2 version: 0.3.2 + undici: + specifier: ^7.4.0 + version: 7.4.0 zod: specifier: ^3.22.4 version: 3.22.4 @@ -62,7 +68,7 @@ importers: version: 9.14.0 tsup: specifier: ^8.3.5 - version: 8.3.5(postcss@8.4.49)(tsx@4.7.0)(typescript@5.3.3)(yaml@2.7.0) + version: 8.3.5(postcss@8.5.3)(tsx@4.7.0)(typescript@5.3.3)(yaml@2.7.0) tsx: specifier: ^4.7.0 version: 4.7.0 @@ -136,13 +142,13 @@ packages: resolution: {integrity: sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==} engines: {node: '>=6.9.0'} - '@babel/parser@7.26.3': - resolution: {integrity: sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==} + '@babel/parser@7.26.9': + resolution: {integrity: sha512-81NWa1njQblgZbQHxWHpxxCzNsa3ZwvFqpUg7P+NNUU6f3UU2jBEg4OlF/J6rl8+PQGh1q6/zWScd001YwcA5A==} engines: {node: '>=6.0.0'} hasBin: true - '@babel/types@7.26.3': - resolution: {integrity: sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==} + '@babel/types@7.26.9': + resolution: {integrity: sha512-Y3IR1cRnOxOCDvMmNiym7XpXQ93iGDDPHx+Zj+NM+rg0fBaShfQLkg+hKPaZCEvg5N/LeCo4+Rj/i3FuJsIQaw==} engines: {node: '>=6.9.0'} '@clack/core@0.3.4': @@ -596,6 +602,12 @@ packages: peerDependencies: eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + '@eslint-community/eslint-utils@4.5.0': + resolution: {integrity: sha512-RoV8Xs9eNwiDvhv7M+xcL4PWyRyIXRY/FLp3buU4h1EYfdF7unWUy3dOjPqb3C7rMUewIcqwW850PgS8h1o1yg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + '@eslint-community/regexpp@4.12.1': resolution: {integrity: sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} @@ -645,6 +657,26 @@ packages: resolution: {integrity: sha512-StAM0vgsD1QK+nFikaKs9Rxe3JGNipiXrpmemNGwM4gWERBXPe9gjzsBoKjgBgq1Vyiy+xy/C652QIWY+MPyYw==} engines: {node: '>=14.0.0', npm: '>=7.0.0'} + '@fuman/fetch@0.0.12': + resolution: {integrity: sha512-3/gyrwHnEwpLqU44tHCedZb63Glp3CaN6mZGz6CqwYtmNyQFJbeS5uFKWXSR2hmL6jj1PdGh7iFUXQHgm+SXAA==} + peerDependencies: + '@badrap/valita': '>=0.4.0' + tough-cookie: ^5.0.0 || ^4.0.0 + valibot: ^0.42.0 + yup: ^1.0.0 + zod: ^3.0.0 + peerDependenciesMeta: + '@badrap/valita': + optional: true + tough-cookie: + optional: true + valibot: + optional: true + yup: + optional: true + zod: + optional: true + '@fuman/io@0.0.8': resolution: {integrity: sha512-+cRZ2JOMYceNQ2Rh6UYro5XVa11j29Sdd3Yhi4GfxAx6ZwCNIw3P80xcTRwCZSfMPLDNN9Etkq7NIc5v9lpItw==} @@ -654,6 +686,9 @@ packages: '@fuman/node@0.0.9': resolution: {integrity: sha512-ImOGEv1T1n/AOqfPH8ag1q/i0RvxnG0EfYVwlfRl/PXW/uNJiH3PRgT4euvXPNlHx6DViDiFn4ADecz9bTrtUg==} + '@fuman/utils@0.0.11': + resolution: {integrity: sha512-hrQhduD3gMXRq1Xz1OMS/6l+Ta8ZUr2hr4d2LLI0JdoSJW31TbvE+QfTuQYvLiQ9pkvYPu1MGnW3Ta3irw16Bw==} + '@fuman/utils@0.0.4': resolution: {integrity: sha512-YBZIlGDbM8s9G85pWFZJ9wQrDY4511XLHZ06/uxZfXBY0eSStwje8JFNmRMNF0SjRk4D3iRmPl9wirHKTkg89w==} @@ -2127,8 +2162,8 @@ packages: mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} - nanoid@3.3.8: - resolution: {integrity: sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==} + nanoid@3.3.9: + resolution: {integrity: sha512-SppoicMGpZvbF1l3z4x7No3OlIjP7QJvC9XR7AhZr1kL133KHnKPztkKDc+Ir4aJ/1VhTySrtKhrsycmrMQfvg==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -2276,8 +2311,8 @@ packages: resolution: {integrity: sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==} engines: {node: '>=4'} - postcss@8.4.49: - resolution: {integrity: sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==} + postcss@8.5.3: + resolution: {integrity: sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==} engines: {node: ^10 || ^12 || >=14} prebuild-install@7.1.2: @@ -2387,6 +2422,11 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.7.1: + resolution: {integrity: sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==} + engines: {node: '>=10'} + hasBin: true + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -2552,8 +2592,8 @@ packages: peerDependencies: typescript: '>=4.2.0' - ts-api-utils@2.0.0: - resolution: {integrity: sha512-xCt/TOAc+EOHS1XPnijD3/yzpH6qg2xppZO1YDqGoVsNXfQfzHpOdNuXwrwOU8u4ITXJyDCTyt8w5g1sZv9ynQ==} + ts-api-utils@2.0.1: + resolution: {integrity: sha512-dnlgjFSVetynI8nzgJ+qF62efpglpWRk8isUEWZGWlJYySCTD6aKvbUDu+zbPeDakk3bg5H4XpitHukgfL1m9w==} engines: {node: '>=18.12'} peerDependencies: typescript: '>=4.8.4' @@ -2624,6 +2664,10 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici@7.4.0: + resolution: {integrity: sha512-PUQM3/es3noM24oUn10u3kNNap0AbxESOmnssmW+dOi9yGwlUSi5nTNYl3bNbTkWOF8YZDkx2tCmj9OtQ3iGGw==} + engines: {node: '>=20.18.1'} + unist-util-is@6.0.0: resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} @@ -2794,11 +2838,11 @@ snapshots: '@babel/helper-validator-identifier@7.25.9': {} - '@babel/parser@7.26.3': + '@babel/parser@7.26.9': dependencies: - '@babel/types': 7.26.3 + '@babel/types': 7.26.9 - '@babel/types@7.26.3': + '@babel/types@7.26.9': dependencies: '@babel/helper-string-parser': 7.25.9 '@babel/helper-validator-identifier': 7.25.9 @@ -3058,6 +3102,11 @@ snapshots: eslint: 9.14.0 eslint-visitor-keys: 3.4.3 + '@eslint-community/eslint-utils@4.5.0(eslint@9.14.0)': + dependencies: + eslint: 9.14.0 + eslint-visitor-keys: 3.4.3 + '@eslint-community/regexpp@4.12.1': {} '@eslint/compat@1.2.2(eslint@9.14.0)': @@ -3109,6 +3158,12 @@ snapshots: '@fluent/langneg@0.7.0': {} + '@fuman/fetch@0.0.12(zod@3.22.4)': + dependencies: + '@fuman/utils': 0.0.11 + optionalDependencies: + zod: 3.22.4 + '@fuman/io@0.0.8': dependencies: '@fuman/utils': 0.0.4 @@ -3124,6 +3179,8 @@ snapshots: '@fuman/net': 0.0.9 '@fuman/utils': 0.0.4 + '@fuman/utils@0.0.11': {} + '@fuman/utils@0.0.4': {} '@humanfs/core@0.19.1': {} @@ -3422,8 +3479,8 @@ snapshots: fast-glob: 3.3.3 is-glob: 4.0.3 minimatch: 9.0.5 - semver: 7.6.3 - ts-api-utils: 2.0.0(typescript@5.3.3) + semver: 7.7.1 + ts-api-utils: 2.0.1(typescript@5.3.3) typescript: 5.3.3 transitivePeerDependencies: - supports-color @@ -3441,7 +3498,7 @@ snapshots: '@typescript-eslint/utils@8.19.1(eslint@9.14.0)(typescript@5.3.3)': dependencies: - '@eslint-community/eslint-utils': 4.4.1(eslint@9.14.0) + '@eslint-community/eslint-utils': 4.5.0(eslint@9.14.0) '@typescript-eslint/scope-manager': 8.19.1 '@typescript-eslint/types': 8.19.1 '@typescript-eslint/typescript-estree': 8.19.1(typescript@5.3.3) @@ -3469,7 +3526,7 @@ snapshots: '@vue/compiler-core@3.5.13': dependencies: - '@babel/parser': 7.26.3 + '@babel/parser': 7.26.9 '@vue/shared': 3.5.13 entities: 4.5.0 estree-walker: 2.0.2 @@ -3482,14 +3539,14 @@ snapshots: '@vue/compiler-sfc@3.5.13': dependencies: - '@babel/parser': 7.26.3 + '@babel/parser': 7.26.9 '@vue/compiler-core': 3.5.13 '@vue/compiler-dom': 3.5.13 '@vue/compiler-ssr': 3.5.13 '@vue/shared': 3.5.13 estree-walker: 2.0.2 magic-string: 0.30.17 - postcss: 8.4.49 + postcss: 8.5.3 source-map-js: 1.2.1 '@vue/compiler-ssr@3.5.13': @@ -4843,7 +4900,7 @@ snapshots: object-assign: 4.1.1 thenify-all: 1.6.0 - nanoid@3.3.8: {} + nanoid@3.3.9: {} napi-build-utils@1.0.2: {} @@ -4956,11 +5013,11 @@ snapshots: pluralize@8.0.0: {} - postcss-load-config@6.0.1(postcss@8.4.49)(tsx@4.7.0)(yaml@2.7.0): + postcss-load-config@6.0.1(postcss@8.5.3)(tsx@4.7.0)(yaml@2.7.0): dependencies: lilconfig: 3.1.2 optionalDependencies: - postcss: 8.4.49 + postcss: 8.5.3 tsx: 4.7.0 yaml: 2.7.0 @@ -4969,9 +5026,9 @@ snapshots: cssesc: 3.0.0 util-deprecate: 1.0.2 - postcss@8.4.49: + postcss@8.5.3: dependencies: - nanoid: 3.3.8 + nanoid: 3.3.9 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -5104,6 +5161,8 @@ snapshots: semver@7.6.3: {} + semver@7.7.1: {} + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -5276,7 +5335,7 @@ snapshots: dependencies: typescript: 5.3.3 - ts-api-utils@2.0.0(typescript@5.3.3): + ts-api-utils@2.0.1(typescript@5.3.3): dependencies: typescript: 5.3.3 @@ -5284,7 +5343,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.3.5(postcss@8.4.49)(tsx@4.7.0)(typescript@5.3.3)(yaml@2.7.0): + tsup@8.3.5(postcss@8.5.3)(tsx@4.7.0)(typescript@5.3.3)(yaml@2.7.0): dependencies: bundle-require: 5.0.0(esbuild@0.24.0) cac: 6.7.14 @@ -5294,7 +5353,7 @@ snapshots: esbuild: 0.24.0 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.4.49)(tsx@4.7.0)(yaml@2.7.0) + postcss-load-config: 6.0.1(postcss@8.5.3)(tsx@4.7.0)(yaml@2.7.0) resolve-from: 5.0.0 rollup: 4.24.4 source-map: 0.8.0-beta.0 @@ -5303,7 +5362,7 @@ snapshots: tinyglobby: 0.2.10 tree-kill: 1.2.2 optionalDependencies: - postcss: 8.4.49 + postcss: 8.5.3 typescript: 5.3.3 transitivePeerDependencies: - jiti @@ -5342,6 +5401,8 @@ snapshots: undici-types@5.26.5: {} + undici@7.4.0: {} + unist-util-is@6.0.0: dependencies: '@types/unist': 3.0.3 diff --git a/src/core/data/cobalt.ts b/src/core/data/cobalt.ts deleted file mode 100644 index 68eb1a3..0000000 --- a/src/core/data/cobalt.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { z } from "zod" - -import type { Result } from "@/core/utils/result" -import { error, ok } from "@/core/utils/result" -import type { Text } from "@/core/utils/text" -import { compound, literal, translatable } from "@/core/utils/text" - -const genericErrorSchema = z.object({ - status: z.literal("error"), - error: z.object({ - code: z.string(), - }), -}) -export type GenericCobaltError = z.infer - -// Main - -const mediaStreamSchema = z.object({ - status: z.literal("tunnel"), - url: z.string().url(), - filename: z.string(), -}) - -const mediaRedirectSchema = z.object({ - status: z.literal("redirect"), - url: z.string().url(), - filename: z.string(), -}) - -const mediaPickerSchema = z.object({ - status: z.literal("picker"), - audio: z.string().url(), - picker: z.array(z.object({ - type: z.union([z.literal("photo"), z.literal("video"), z.literal("gif")]), - url: z.string().url(), - })), -}) - -const mediaResponseSchema = z.discriminatedUnion("status", [ - mediaStreamSchema, - mediaRedirectSchema, - mediaPickerSchema, - genericErrorSchema, -]) - -export type CobaltMediaResponse = z.infer -export type SuccessfulCobaltMediaResponse = Exclude - -const cobaltErrors = new Map([ - ["service.unsupported", "error-invalid-url"], - ["service.disabled", "error-invalid-url"], - ["link.invalid", "error-invalid-url"], - ["link.unsupported", "error-invalid-url"], - - ["content.too_long", "error-too-large"], - - ["content.video.unavailable", "error-media-unavailable"], - ["content.video.live", "error-media-unavailable"], - ["content.video.private", "error-media-unavailable"], - ["content.video.age", "error-media-unavailable"], - ["content.video.region", "error-media-unavailable"], - ["content.post.unavailable", "error-media-unavailable"], - ["content.post.private", "error-media-unavailable"], - ["content.post.age", "error-media-unavailable"], -].map(([k, v]) => [`error.api.${k}`, v])) - -const stackHeaders = (...headers: ([string, string] | null | false | undefined)[]) => headers.filter((h): h is [string, string] => !!h) - -const stackObjects = (...objs: (T | null | false | undefined)[]) => objs.reduce( - (p, c) => c ? ({ ...p, ...c }) : p, - {}, -) - -export async function fetchMedia({ url, lang, apiBaseUrl, downloadMode = "auto", auth, youtubeHls }: { - url: string, - lang?: string, - downloadMode?: string, - apiBaseUrl: string, - auth?: string, - youtubeHls?: boolean, -}): Promise> { - const res = await fetch(`${apiBaseUrl}`, { - method: "POST", - headers: stackHeaders( - ["Accept", "application/json"], - ["Content-Type", "application/json"], - ["User-Agent", "cobold (+https://github.com/tskau/cobold)"], - !!auth && ["Authorization", auth], - !!lang && ["Accept-Language", lang], - ), - body: JSON.stringify(stackObjects( - { url, downloadMode, filenameStyle: "basic" }, - youtubeHls && { youtubeHLS: true }, - )), - }).catch(() => null) - - if (!res) - return error(translatable("error-unresponsive")) - const body = await res.json().catch(() => null) - - const data = mediaResponseSchema.safeParse(body) - if (!data.success) - return error(translatable("error-invalid-response")) - - if (data.data.status === "error") { - const code = data.data.error.code - const errorKey = cobaltErrors.get(code) - if (errorKey) - return error(translatable(errorKey)) - return error(compound(translatable("error-invalid-response"), literal(` [${code}]`))) - } - - return ok(data.data) -} - -// Stream - -export async function fetchStream(url: string) { - const data = await fetch(url, { - headers: [ - ["User-Agent", "cobold (+https://github.com/tskau/cobold)"], - ], - }) - - if (!data.ok) { - const error = await data.json().catch(() => null) as unknown - const body = genericErrorSchema.safeParse(error) - if (!body.success) { - throw new Error(`streaming from ${new URL(url).host} failed`) - } - return body.data - } - - const buffer = await data.arrayBuffer() - if (!buffer.byteLength) - throw new Error(`empty body from ${new URL(url).host}`) - - return { - status: "success" as const, - buffer, - } -} diff --git a/src/core/data/cobalt/common.ts b/src/core/data/cobalt/common.ts new file mode 100644 index 0000000..ae97f99 --- /dev/null +++ b/src/core/data/cobalt/common.ts @@ -0,0 +1,15 @@ +import { createFfetch, ffetchAddons } from "@fuman/fetch" +import { ffetchZodAdapter } from "@fuman/fetch/zod" + +import { proxyAddon } from "@/core/utils/proxy" + +export const baseFetch = createFfetch({ + addons: [ + ffetchAddons.parser(ffetchZodAdapter()), + proxyAddon(), + ], + headers: [ + ["User-Agent", "cobold (+https://github.com/tskau/cobold)"], + ], + validateResponse: false, +}) diff --git a/src/core/data/cobalt/download.ts b/src/core/data/cobalt/download.ts new file mode 100644 index 0000000..46b75f9 --- /dev/null +++ b/src/core/data/cobalt/download.ts @@ -0,0 +1,95 @@ +import { z } from "zod" + +import { baseFetch } from "@/core/data/cobalt/common" +import type { GenericCobaltError } from "@/core/data/cobalt/error" +import { cobaltErrors, genericErrorSchema } from "@/core/data/cobalt/error" + +import { merge, stack } from "@/core/utils/compose" + +import type { Result } from "@/core/utils/result" +import { error, ok } from "@/core/utils/result" + +import type { Text } from "@/core/utils/text" +import { compound, literal, translatable } from "@/core/utils/text" + +const mediaStreamSchema = z.object({ + status: z.literal("tunnel"), + url: z.string().url(), + filename: z.string(), +}) + +const mediaRedirectSchema = z.object({ + status: z.literal("redirect"), + url: z.string().url(), + filename: z.string(), +}) + +const mediaPickerSchema = z.object({ + status: z.literal("picker"), + audio: z.string().url(), + picker: z.array(z.object({ + type: z.union([z.literal("photo"), z.literal("video"), z.literal("gif")]), + url: z.string().url(), + })), +}) + +const mediaResponseSchema = z.discriminatedUnion("status", [ + mediaStreamSchema, + mediaRedirectSchema, + mediaPickerSchema, + genericErrorSchema, +]) + +export type CobaltMediaResponse = z.infer +export type SuccessfulCobaltMediaResponse = Exclude + +const ffetch = baseFetch.extend({ + headers: [ + ["Accept", "application/json"], + ["Content-Type", "application/json"], + ], +}) + +export async function startDownload({ url, lang, apiBaseUrl, downloadMode = "auto", auth, youtubeHls, proxy }: { + url: string, + lang?: string, + downloadMode?: string, + apiBaseUrl: string, + auth?: string, + youtubeHls?: boolean, + proxy?: string, +}): Promise> { + const res = await ffetch("/", { + method: "POST", + headers: stack<[string, string]>( + !!auth && ["Authorization", auth], + !!lang && ["Accept-Language", lang], + ), + json: merge( + { url, downloadMode, filenameStyle: "basic" }, + youtubeHls && { youtubeHLS: true }, + ), + baseUrl: apiBaseUrl, + proxy: proxy || undefined, + }) + .safelyParsedJson(mediaResponseSchema) + .catch((e) => { + console.log(e) + return null + }) + + if (!res) + return error(translatable("error-unresponsive")) + if (!res.success) + return error(translatable("error-invalid-response")) + + if (res.data.status === "error") { + const code = res.data.error.code + const errorKey = cobaltErrors.get(code) + if (errorKey) + return error(translatable(errorKey)) + return error(compound(translatable("error-invalid-response"), literal(` [${code}]`))) + } + + return ok(res.data) +} diff --git a/src/core/data/cobalt/error.ts b/src/core/data/cobalt/error.ts new file mode 100644 index 0000000..603aea0 --- /dev/null +++ b/src/core/data/cobalt/error.ts @@ -0,0 +1,27 @@ +import { z } from "zod" + +export const genericErrorSchema = z.object({ + status: z.literal("error"), + error: z.object({ + code: z.string(), + }), +}) +export type GenericCobaltError = z.infer + +export const cobaltErrors = new Map([ + ["service.unsupported", "error-invalid-url"], + ["service.disabled", "error-invalid-url"], + ["link.invalid", "error-invalid-url"], + ["link.unsupported", "error-invalid-url"], + + ["content.too_long", "error-too-large"], + + ["content.video.unavailable", "error-media-unavailable"], + ["content.video.live", "error-media-unavailable"], + ["content.video.private", "error-media-unavailable"], + ["content.video.age", "error-media-unavailable"], + ["content.video.region", "error-media-unavailable"], + ["content.post.unavailable", "error-media-unavailable"], + ["content.post.private", "error-media-unavailable"], + ["content.post.age", "error-media-unavailable"], +].map(([k, v]) => [`error.api.${k}`, v])) diff --git a/src/core/data/cobalt/external.ts b/src/core/data/cobalt/external.ts new file mode 100644 index 0000000..ea737a8 --- /dev/null +++ b/src/core/data/cobalt/external.ts @@ -0,0 +1,5 @@ +import { baseFetch } from "@/core/data/cobalt/common" + +export async function retrieveExternalMedia(url: string, proxy?: string) { + return baseFetch(url, { proxy }).then(r => r.arrayBuffer()) +} diff --git a/src/core/data/cobalt/index.ts b/src/core/data/cobalt/index.ts new file mode 100644 index 0000000..b4774e0 --- /dev/null +++ b/src/core/data/cobalt/index.ts @@ -0,0 +1,5 @@ +export { type CobaltMediaResponse, startDownload, type SuccessfulCobaltMediaResponse } from "@/core/data/cobalt/download" +export type { GenericCobaltError } from "@/core/data/cobalt/error" +export { retrieveExternalMedia } from "@/core/data/cobalt/external" +export { type ApiServer, apiServerSchema } from "@/core/data/cobalt/server" +export { retrieveTunneledMedia } from "@/core/data/cobalt/tunnel" diff --git a/src/core/data/cobalt/server.ts b/src/core/data/cobalt/server.ts new file mode 100644 index 0000000..e4df4be --- /dev/null +++ b/src/core/data/cobalt/server.ts @@ -0,0 +1,25 @@ +import { z } from "zod" +import { urlWithAuthSchema } from "@/core/utils/url" + +export const apiServerSchema = z.object({ + name: z.string().optional(), + url: z.string().url(), + auth: z.string().optional(), + youtubeHls: z.boolean().optional(), + unsafe: z.boolean().optional(), + proxy: z.string().url().optional(), +}).or( + urlWithAuthSchema.transform(data => ({ + name: undefined, + url: data.url, + auth: data.auth, + youtubeHls: undefined, + unsafe: undefined, + proxy: undefined, + })), +).transform(data => ({ + ...data, + name: data.name ?? data.url, +})) + +export type ApiServer = z.infer diff --git a/src/core/data/cobalt/tunnel.ts b/src/core/data/cobalt/tunnel.ts new file mode 100644 index 0000000..9b5a835 --- /dev/null +++ b/src/core/data/cobalt/tunnel.ts @@ -0,0 +1,24 @@ +import { baseFetch } from "@/core/data/cobalt/common" +import { genericErrorSchema } from "@/core/data/cobalt/error" + +export async function retrieveTunneledMedia(url: string, proxy?: string) { + const data = await baseFetch(url, { proxy }) + + if (!data.ok) { + const error = await data.json().catch(() => null) as unknown + const body = genericErrorSchema.safeParse(error) + if (!body.success) { + throw new Error(`streaming from ${new URL(url).host} failed`) + } + return body.data + } + + const buffer = await data.arrayBuffer() + if (!buffer.byteLength) + throw new Error(`empty body from ${new URL(url).host}`) + + return { + status: "success" as const, + buffer, + } +} diff --git a/src/core/data/request.ts b/src/core/data/request.ts index 807e0c5..07d9eb7 100644 --- a/src/core/data/request.ts +++ b/src/core/data/request.ts @@ -4,36 +4,16 @@ import { randomUUID } from "node:crypto" import { eq } from "drizzle-orm" import { z } from "zod" -import type { SuccessfulCobaltMediaResponse } from "@/core/data/cobalt" -import { fetchMedia, fetchStream } from "@/core/data/cobalt" +import type { ApiServer, SuccessfulCobaltMediaResponse } from "@/core/data/cobalt" +import { retrieveExternalMedia, retrieveTunneledMedia, startDownload } from "@/core/data/cobalt" + import { db } from "@/core/data/db/database" import { requests } from "@/core/data/db/schema" import type { Result } from "@/core/utils/result" import { error, ok } from "@/core/utils/result" import type { CompoundText, Text } from "@/core/utils/text" import { compound, literal, translatable } from "@/core/utils/text" -import { safeUrlSchema, urlWithAuthSchema } from "@/core/utils/url" - -export const apiServerSchema = z.object({ - name: z.string().optional(), - url: z.string().url(), - auth: z.string().optional(), - youtubeHls: z.boolean().optional(), - unsafe: z.boolean().optional(), -}).or( - urlWithAuthSchema.transform(data => ({ - name: undefined, - url: data.url, - auth: data.auth, - youtubeHls: undefined, - unsafe: undefined, - })), -).transform(data => ({ - ...data, - name: data.name ?? data.url, -})) - -export type ApiServer = z.infer +import { safeUrlSchema } from "@/core/utils/url" const mediaUrlSchema = z.string().url() function tryParseUrl(url: string) { @@ -74,8 +54,6 @@ export async function createRequest(userInput: string, authorId: number): Promis export const getRequest = (requestId: string) => db.query.requests.findFirst({ where: eq(requests.id, requestId) }) -const retrieveMedia = async (url: string) => fetch(url).then(r => r.arrayBuffer()) - export type OutputMedia = { type: "single", fileName?: string, file: ArrayBuffer } | { type: "multiple", files: ArrayBuffer[] } export const outputOptions = ["auto", "audio"] export async function finishRequest(outputType: string, request: MediaRequest, apiPool: ApiServer[], lang?: string): Promise> { @@ -86,7 +64,7 @@ export async function finishRequest(outputType: string, request: MediaRequest, a return res if (res.result.status === "tunnel") { - const data = await fetchStream(res.result.url) + const data = await retrieveTunneledMedia(res.result.url, res.result.api.proxy) if (data.status === "error") return error(translatable(data.error.code)) @@ -100,7 +78,7 @@ export async function finishRequest(outputType: string, request: MediaRequest, a if (res.result.status === "picker") { if (outputType === "audio") { const source = new URL(res.result.audio) - const buffer = await retrieveMedia(source.href) + const buffer = await retrieveExternalMedia(source.href) return ok({ type: "single", fileName: source.pathname.split("/").at(-1), @@ -108,7 +86,7 @@ export async function finishRequest(outputType: string, request: MediaRequest, a }) } if (res.result.picker.length !== 1) { - const files = await Promise.all(res.result.picker.map(i => retrieveMedia(i.url))) + const files = await Promise.all(res.result.picker.map(i => retrieveExternalMedia(i.url))) return ok({ type: "multiple", files, @@ -116,7 +94,7 @@ export async function finishRequest(outputType: string, request: MediaRequest, a } const file = res.result.picker[0] const source = new URL(file.url) - const buffer = await retrieveMedia(source.href) + const buffer = await retrieveExternalMedia(source.href, res.result.api.proxy) return ok({ type: "single", fileName: source.pathname.split("/").at(-1), @@ -124,7 +102,7 @@ export async function finishRequest(outputType: string, request: MediaRequest, a }) } - const buffer = await retrieveMedia(res.result.url) + const buffer = await retrieveExternalMedia(res.result.url, res.result.api.proxy) return ok({ type: "single", fileName: res.result.filename, @@ -132,7 +110,7 @@ export async function finishRequest(outputType: string, request: MediaRequest, a }) } -async function tryDownload(outputType: string, request: MediaRequest, apiPool: ApiServer[], lang?: string, fails: Text[] = []): Promise> { +async function tryDownload(outputType: string, request: MediaRequest, apiPool: ApiServer[], lang?: string, fails: Text[] = []): Promise> { const currentApi = apiPool.at(0) if (!currentApi) return error(compound(...fails)) @@ -147,13 +125,14 @@ async function tryDownload(outputType: string, request: MediaRequest, apiPool: A ) } - const res = await fetchMedia({ + const res = await startDownload({ url: request.url, downloadMode: outputType, lang, apiBaseUrl: currentApi.url, auth: currentApi.auth, youtubeHls: currentApi.youtubeHls, + proxy: currentApi.proxy, }) if (!res.success) { @@ -182,5 +161,5 @@ async function tryDownload(outputType: string, request: MediaRequest, apiPool: A } } - return res + return { success: res.success, result: { ...res.result, api: currentApi } } } diff --git a/src/core/utils/compose.ts b/src/core/utils/compose.ts new file mode 100644 index 0000000..ed3fb0b --- /dev/null +++ b/src/core/utils/compose.ts @@ -0,0 +1,8 @@ +export const stack = (...objects: (T | null | false | undefined)[]) => + objects.filter((h): h is T => !!h) + +export const merge = (...objs: (T | null | false | undefined)[]) => + objs.reduce( + (p, c) => c ? ({ ...p, ...c }) : p, + {}, + ) diff --git a/src/core/utils/proxy.ts b/src/core/utils/proxy.ts new file mode 100644 index 0000000..d829c1f --- /dev/null +++ b/src/core/utils/proxy.ts @@ -0,0 +1,15 @@ +import type { FfetchAddon } from "@fuman/fetch" +import { ProxyAgent } from "undici" + +export const proxyAddon = (): FfetchAddon<{ proxy?: string | false }, object> => { + return { + beforeRequest: (ctx) => { + if (ctx.options.proxy) { + ctx.options.extra ??= {} + // @ts-expect-error This API is only available with undici and is not included in default Request types + ctx.options.extra.dispatcher + = new ProxyAgent(ctx.options.proxy) + } + }, + } +} diff --git a/src/telegram/helpers/env.ts b/src/telegram/helpers/env.ts index c28ca29..a90fc53 100644 --- a/src/telegram/helpers/env.ts +++ b/src/telegram/helpers/env.ts @@ -4,7 +4,7 @@ import { createEnv } from "@t3-oss/env-core" import { config } from "dotenv" import { z } from "zod" -import { apiServerSchema } from "@/core/data/request" +import { apiServerSchema } from "@/core/data/cobalt" config() @@ -28,6 +28,7 @@ export const env = createEnv({ .pipe(z.array(apiServerSchema)), SELECT_TYPE_PHOTO_URL: z.string().url().default("https://i.otomir23.me/buckets/cobold/download.png"), ERROR_CHAT_ID: z.coerce.number().int().optional(), + CUSTOM_INSTANCE_PROXY_URL: z.string().url().optional(), }, emptyStringAsUndefined: true, runtimeEnv: process.env, diff --git a/src/telegram/helpers/handler.ts b/src/telegram/helpers/handler.ts index 7733739..cf3b05a 100644 --- a/src/telegram/helpers/handler.ts +++ b/src/telegram/helpers/handler.ts @@ -4,7 +4,8 @@ import type { GeneralTrack, ImageTrack, VideoTrack } from "mediainfo.js" import { CallbackDataBuilder } from "@mtcute/dispatcher" import mediaInfoFactory from "mediainfo.js" -import type { ApiServer, MediaRequest } from "@/core/data/request" +import type { ApiServer } from "@/core/data/cobalt" +import type { MediaRequest } from "@/core/data/request" import { finishRequest, outputOptions } from "@/core/data/request" import type { Result } from "@/core/utils/result" import { error, ok } from "@/core/utils/result" @@ -91,7 +92,7 @@ export async function handleMediaDownload(outputType: string, request: MediaRequ const settings = await getPeerSettings(peer) const locale = settings.languageOverride ?? getPeerLocale(peer) const endpoints: ApiServer[] = settings.instanceOverride - ? [{ name: "custom", ...urlWithAuthSchema.parse(settings.instanceOverride), unsafe: true }] + ? [{ name: "custom", ...urlWithAuthSchema.parse(settings.instanceOverride), unsafe: true, proxy: env.CUSTOM_INSTANCE_PROXY_URL }] : env.API_ENDPOINTS const res = await finishRequest(outputType, request, endpoints, locale) if (!res.success)