diff --git a/.changeset/quiet-vans-accept.md b/.changeset/quiet-vans-accept.md new file mode 100644 index 0000000..1b05a94 --- /dev/null +++ b/.changeset/quiet-vans-accept.md @@ -0,0 +1,5 @@ +--- +"@embedly/platforms": minor +--- + +added reddit support (mostly working) diff --git a/apps/api/biome.json b/apps/api/biome.json index 8aeac5f..98ebfcf 100644 --- a/apps/api/biome.json +++ b/apps/api/biome.json @@ -1,7 +1,7 @@ { "root": false, "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"], + "extends": ["@embedly/config/biome.json"], "files": { "includes": ["./src/**"] } diff --git a/apps/api/package.json b/apps/api/package.json index 0d8135d..9911d32 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -15,7 +15,7 @@ "scripts": { "dev": "wrangler dev --config wrangler.local.jsonc", "lint": "biome ci ./src/*", - "build": "pkgroll && wrangler deploy --dry-run", + "build": "pkgroll --sourcemap=inline && tsc && wrangler deploy --dry-run", "deploy": "wrangler deploy --minify", "deploy:local": "wrangler deploy --config wrangler.local.jsonc --minify", "cf-typegen": "wrangler types" diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json index b9795cc..8d36424 100644 --- a/apps/api/tsconfig.json +++ b/apps/api/tsconfig.json @@ -1,8 +1,9 @@ { - "extends": "@embedly/config/typescript", + "extends": "@embedly/config/tsconfig.json", "exclude": ["node_modules"], "include": ["src"], "compilerOptions": { "types": ["./worker-configuration.d.ts", "@types/node"], + "outDir": "./dist" } } diff --git a/apps/bot/biome.json b/apps/bot/biome.json index 05d8939..167b18f 100644 --- a/apps/bot/biome.json +++ b/apps/bot/biome.json @@ -1,5 +1,5 @@ { "root": false, "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"] + "extends": ["@embedly/config/biome.json"] } diff --git a/apps/bot/package.json b/apps/bot/package.json index 475ff51..6db5573 100644 --- a/apps/bot/package.json +++ b/apps/bot/package.json @@ -15,7 +15,7 @@ "scripts": { "dev": "tsx watch --env-file=.env src/main.ts", "lint": "biome ci ./src/*", - "build": "pkgroll" + "build": "pkgroll --sourcemap=inline && tsc" }, "devDependencies": { "@biomejs/biome": "2.3.10", diff --git a/apps/bot/tsconfig.json b/apps/bot/tsconfig.json index 87d9ae3..07fa614 100644 --- a/apps/bot/tsconfig.json +++ b/apps/bot/tsconfig.json @@ -1,5 +1,8 @@ { - "extends": "@embedly/config/typescript", + "extends": "@embedly/config/tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + }, "exclude": ["node_modules"], "include": ["src"] } diff --git a/biome.json b/biome.json index fa578a4..33e068c 100644 --- a/biome.json +++ b/biome.json @@ -1,5 +1,5 @@ { "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"], + "extends": ["@embedly/config/biome.json"], "root": true } diff --git a/packages/builder/biome.json b/packages/builder/biome.json index 05d8939..167b18f 100644 --- a/packages/builder/biome.json +++ b/packages/builder/biome.json @@ -1,5 +1,5 @@ { "root": false, "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"] + "extends": ["@embedly/config/biome.json"] } diff --git a/packages/builder/package.json b/packages/builder/package.json index 555ff2b..ad7f535 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -14,7 +14,7 @@ }, "scripts": { "lint": "biome ci ./src/*", - "build": "pkgroll" + "build": "pkgroll --sourcemap=inline && tsc && tsc" }, "devDependencies": { "@biomejs/biome": "2.3.10", diff --git a/packages/builder/src/Embed.ts b/packages/builder/src/Embed.ts index 8bf15ad..89a0041 100644 --- a/packages/builder/src/Embed.ts +++ b/packages/builder/src/Embed.ts @@ -286,7 +286,7 @@ export class Embed implements EmbedData { if (description) { section.addTextDisplayComponents((builder) => - builder.setContent(escapeMarkdown(description)) + builder.setContent(description) ); } diff --git a/packages/builder/tsconfig.json b/packages/builder/tsconfig.json index 65521da..3a138a9 100644 --- a/packages/builder/tsconfig.json +++ b/packages/builder/tsconfig.json @@ -1,7 +1,8 @@ { - "extends": "@embedly/config/typescript", + "extends": "@embedly/config/tsconfig.json", "compilerOptions": { - "lib": ["WebWorker", "ESNext"] + "lib": ["WebWorker", "ESNext"], + "outDir": "./dist" }, "exclude": ["node_modules"], "include": ["src"] diff --git a/packages/config/package.json b/packages/config/package.json index 91f1059..b15c030 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -4,8 +4,8 @@ "private": true, "type": "module", "exports": { - "./typescript": "./tsconfig.json", - "./biome": "./biome.json" + "./tsconfig.json": "./tsconfig.json", + "./biome.json": "./biome.json" }, "devDependencies": { "@biomejs/biome": "2.3.10" diff --git a/packages/config/tsconfig.json b/packages/config/tsconfig.json index b601a8b..b9357a1 100644 --- a/packages/config/tsconfig.json +++ b/packages/config/tsconfig.json @@ -5,13 +5,13 @@ "alwaysStrict": true, "declaration": true, "declarationMap": true, + "emitDeclarationOnly": true, "esModuleInterop": true, "importHelpers": false, "lib": ["esnext"], "module": "NodeNext", "moduleResolution": "NodeNext", "newLine": "lf", - "noEmit": true, "noEmitHelpers": false, "noFallthroughCasesInSwitch": true, "noImplicitReturns": true, @@ -21,6 +21,8 @@ "pretty": true, "removeComments": false, "resolveJsonModule": true, + "skipDefaultLibCheck": true, + "skipLibCheck": true, "sourceMap": true, "strict": true, "target": "ESNext", diff --git a/packages/logging/biome.json b/packages/logging/biome.json index 05d8939..167b18f 100644 --- a/packages/logging/biome.json +++ b/packages/logging/biome.json @@ -1,5 +1,5 @@ { "root": false, "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"] + "extends": ["@embedly/config/biome.json"] } diff --git a/packages/logging/package.json b/packages/logging/package.json index 283e403..70c36f3 100644 --- a/packages/logging/package.json +++ b/packages/logging/package.json @@ -14,7 +14,7 @@ }, "scripts": { "lint": "biome ci ./src/*", - "build": "pkgroll" + "build": "pkgroll --sourcemap=inline && tsc" }, "devDependencies": { "@biomejs/biome": "2.3.10", diff --git a/packages/logging/tsconfig.json b/packages/logging/tsconfig.json index 65521da..3a138a9 100644 --- a/packages/logging/tsconfig.json +++ b/packages/logging/tsconfig.json @@ -1,7 +1,8 @@ { - "extends": "@embedly/config/typescript", + "extends": "@embedly/config/tsconfig.json", "compilerOptions": { - "lib": ["WebWorker", "ESNext"] + "lib": ["WebWorker", "ESNext"], + "outDir": "./dist" }, "exclude": ["node_modules"], "include": ["src"] diff --git a/packages/parser/biome.json b/packages/parser/biome.json index 05d8939..167b18f 100644 --- a/packages/parser/biome.json +++ b/packages/parser/biome.json @@ -1,5 +1,5 @@ { "root": false, "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"] + "extends": ["@embedly/config/biome.json"] } diff --git a/packages/parser/package.json b/packages/parser/package.json index 65d0347..9b00ebb 100644 --- a/packages/parser/package.json +++ b/packages/parser/package.json @@ -14,7 +14,7 @@ }, "scripts": { "lint": "biome ci ./src/*", - "build": "pkgroll" + "build": "pkgroll --sourcemap=inline && tsc" }, "devDependencies": { "@biomejs/biome": "2.3.10", diff --git a/packages/parser/src/parse.ts b/packages/parser/src/parse.ts index e6652b9..7cfe006 100644 --- a/packages/parser/src/parse.ts +++ b/packages/parser/src/parse.ts @@ -3,6 +3,7 @@ import { CBC_REGEX, GENERIC_LINK_REGEX, IG_REGEX, + REDDIT_REGEX, THREADS_REGEX, TIKTOK_REGEX_MAIN, TWITTER_REGEX @@ -37,5 +38,8 @@ export function getPlatformFromURL( if (THREADS_REGEX.test(url)) { return { type: EmbedlyPlatformType.Threads }; } + if (REDDIT_REGEX.test(url)) { + return { type: EmbedlyPlatformType.Reddit }; + } return null; } diff --git a/packages/parser/src/regex.ts b/packages/parser/src/regex.ts index 8e26ee0..7688a73 100644 --- a/packages/parser/src/regex.ts +++ b/packages/parser/src/regex.ts @@ -11,3 +11,5 @@ export const TIKTOK_REGEX = export const THREADS_REGEX = /threads\.com\/@.*\/post\/(?[A-Za-z0-9-_]+)/; export const CBC_REGEX = /cbc.ca\/.*(?\d\.\d+)/; +export const REDDIT_REGEX = + /https?:\/\/(?:www\.|old\.|m\.)?reddit\.com\/r\/(?\w+)\/comments\/(?[a-z0-9]+)/; diff --git a/packages/parser/tsconfig.json b/packages/parser/tsconfig.json index 65521da..3a138a9 100644 --- a/packages/parser/tsconfig.json +++ b/packages/parser/tsconfig.json @@ -1,7 +1,8 @@ { - "extends": "@embedly/config/typescript", + "extends": "@embedly/config/tsconfig.json", "compilerOptions": { - "lib": ["WebWorker", "ESNext"] + "lib": ["WebWorker", "ESNext"], + "outDir": "./dist" }, "exclude": ["node_modules"], "include": ["src"] diff --git a/packages/platforms/biome.json b/packages/platforms/biome.json index 38b03fc..79be7a7 100644 --- a/packages/platforms/biome.json +++ b/packages/platforms/biome.json @@ -1,7 +1,7 @@ { "root": false, "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"], + "extends": ["@embedly/config/biome.json"], "linter": { "rules": { "complexity": { diff --git a/packages/platforms/package.json b/packages/platforms/package.json index e7f29c1..f9928ba 100644 --- a/packages/platforms/package.json +++ b/packages/platforms/package.json @@ -14,7 +14,7 @@ }, "scripts": { "lint": "biome ci ./src/*", - "build": "pkgroll" + "build": "pkgroll --sourcemap=inline && tsc" }, "devDependencies": { "@biomejs/biome": "2.3.10", diff --git a/packages/platforms/src/Instagram.ts b/packages/platforms/src/Instagram.ts index 2a927d6..bae7742 100644 --- a/packages/platforms/src/Instagram.ts +++ b/packages/platforms/src/Instagram.ts @@ -54,6 +54,10 @@ export class Instagram extends EmbedlyPlatform { "X-FB-LSD": "AVqbxe3J_YA", "X-ASBD-ID": "129477", "Sec-Fetch-Site": "same-origin" + }, + cf: { + cacheTtl: 60 * 60 * 24, + cacheEverything: true } }); if (!resp.ok) { diff --git a/packages/platforms/src/Platform.ts b/packages/platforms/src/Platform.ts index ca0a858..5edae6c 100644 --- a/packages/platforms/src/Platform.ts +++ b/packages/platforms/src/Platform.ts @@ -44,6 +44,8 @@ export abstract class EmbedlyPlatform { }); } abstract fetchPost(post_id: string, env?: any): Promise; - abstract transformRawData(raw_data: any): BaseEmbedData; + abstract transformRawData( + raw_data: any + ): Promise | BaseEmbedData; abstract createEmbed(post_data: T): Promise | Embed; } diff --git a/packages/platforms/src/Reddit.ts b/packages/platforms/src/Reddit.ts new file mode 100644 index 0000000..d4b4d78 --- /dev/null +++ b/packages/platforms/src/Reddit.ts @@ -0,0 +1,159 @@ +import { Embed } from "@embedly/builder"; +import { + EMBEDLY_FAILED_PLATFORM, + EMBEDLY_FETCH_PLATFORM +} from "@embedly/logging"; +import { REDDIT_REGEX } from "@embedly/parser"; +import { + type BaseEmbedData, + EmbedlyPlatformType +} from "@embedly/types"; +import { EmbedlyPlatform } from "./Platform.ts"; + +export class Reddit extends EmbedlyPlatform { + constructor() { + super(EmbedlyPlatformType.Reddit, "reddit", { + fetching: EMBEDLY_FETCH_PLATFORM(EmbedlyPlatformType.Reddit), + failed: EMBEDLY_FAILED_PLATFORM(EmbedlyPlatformType.Reddit) + }); + } + + async parsePostId(url: string): Promise { + const match = REDDIT_REGEX.exec(url)!; + const { post_id, subreddit } = match.groups!; + return `${subreddit}/${post_id}`; + } + + async fetchPost( + post_id: string, + env: { EMBED_USER_AGENT: string } + ): Promise { + const [subreddit, reddit_id] = post_id.split("/"); + const resp = await fetch( + `https://www.reddit.com/r/${subreddit}/comments/${reddit_id}.json?raw_json=1`, + { + method: "GET", + headers: { + "User-Agent": env.EMBED_USER_AGENT + }, + cf: { + cacheTtl: 60 * 60 * 24, + cacheEverything: true + } + } + ); + + if (!resp.ok) { + throw { code: resp.status, message: resp.statusText }; + } + + const post_data = (await resp.json()) as Record; + + const profile_resp = await fetch( + `https://www.reddit.com/user/${post_data.author}/about.json?raw_json=1`, + { + method: "GET", + headers: { + "User-Agent": env.EMBED_USER_AGENT + }, + cf: { + cacheTtl: 60 * 60 * 24, + cacheEverything: true + } + } + ); + + if (!profile_resp.ok) { + throw { code: resp.status, message: resp.statusText }; + } + + const { data: profile_data } = + (await profile_resp.json()) as Record; + + return { + post_data: post_data[0].data.children[0].data, + profile_data + }; + } + + parsePostMedia( + post_data: Record + ): Parameters[0] { + if (post_data.domain === "i.redd.it") { + return [ + { + media: { + url: post_data.url_overridden_by_dest + } + } + ]; + } + if (post_data.media_metadata) { + return Object.values(post_data.media_metadata).map( + (media: any) => ({ + media: { + url: media.s.u + } + }) + ); + } + + if (post_data.preview?.enabled) { + return post_data.preview.images.map((media: any) => ({ + media: { + url: media.source.url + } + })); + } + + if (post_data.media?.reddit_video) { + return [ + { + media: { + url: post_data.media.reddit_video.fallback_url + } + } + ]; + } + + return []; + } + + async transformRawData({ + post_data, + profile_data + }: any): Promise { + return { + platform: this.name, + username: post_data.author, + name: post_data.subreddit_name_prefixed, + profile_url: `https://reddit.com/user/${post_data.author}`, + avatar_url: profile_data.icon_img, + timestamp: post_data.created_utc, + url: `https://reddit.com/${post_data.permalink}`, + description: `## ${post_data.title}\n${post_data.selftext}` + }; + } + + async createEmbed(reddit_data: any): Promise { + const data = await this.transformRawData(reddit_data); + const media = this.parsePostMedia(reddit_data.post_data); + if ( + media.length === 0 && + reddit_data.post_data.url_overridden_by_dest + ) { + data.description += `\n${reddit_data.post_data.url_overridden_by_dest}`; + } + + const embed = new Embed(data); + + if (media.length > 10) { + media.length = 10; + } + if (media.length > 0) { + embed.setMedia(media); + } + + return embed; + } +} diff --git a/packages/platforms/src/main.ts b/packages/platforms/src/main.ts index 7200176..a3560a2 100644 --- a/packages/platforms/src/main.ts +++ b/packages/platforms/src/main.ts @@ -1,6 +1,7 @@ import { EmbedlyPlatformType } from "@embedly/types"; import { CBC } from "./CBC.ts"; import { Instagram } from "./Instagram.ts"; +import { Reddit } from "./Reddit.ts"; import { Threads } from "./Threads.ts"; import { TikTok } from "./TikTok.ts"; import { Twitter } from "./Twitter.ts"; @@ -10,5 +11,6 @@ export default { [EmbedlyPlatformType.Instagram]: new Instagram(), [EmbedlyPlatformType.TikTok]: new TikTok(), [EmbedlyPlatformType.CBC]: new CBC(), - [EmbedlyPlatformType.Threads]: new Threads() + [EmbedlyPlatformType.Threads]: new Threads(), + [EmbedlyPlatformType.Reddit]: new Reddit() }; diff --git a/packages/platforms/tsconfig.json b/packages/platforms/tsconfig.json index 4fca059..2b6f64f 100644 --- a/packages/platforms/tsconfig.json +++ b/packages/platforms/tsconfig.json @@ -1,8 +1,9 @@ { - "extends": "@embedly/config/typescript", + "extends": "@embedly/config/tsconfig.json", "compilerOptions": { "lib": ["WebWorker", "ESNext"], - "types": ["@cloudflare/workers-types", "@types/node"] + "types": ["@cloudflare/workers-types", "@types/node"], + "outDir": "./dist" }, "exclude": ["node_modules"], "include": ["src"] diff --git a/packages/types/biome.json b/packages/types/biome.json index 05d8939..167b18f 100644 --- a/packages/types/biome.json +++ b/packages/types/biome.json @@ -1,5 +1,5 @@ { "root": false, "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", - "extends": ["@embedly/config/biome"] + "extends": ["@embedly/config/biome.json"] } diff --git a/packages/types/package.json b/packages/types/package.json index d5613df..646c070 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -14,7 +14,7 @@ }, "scripts": { "lint": "biome ci ./src/*", - "build": "pkgroll" + "build": "pkgroll --sourcemap=inline && tsc" }, "devDependencies": { "@biomejs/biome": "2.3.10", diff --git a/packages/types/src/main.ts b/packages/types/src/main.ts index 221a9e8..b01958b 100644 --- a/packages/types/src/main.ts +++ b/packages/types/src/main.ts @@ -5,7 +5,8 @@ export enum EmbedlyPlatformType { Instagram = "Instagram", TikTok = "TikTok", CBC = "cbc.ca", - Threads = "Threads" + Threads = "Threads", + Reddit = "Reddit" } export const EmbedlyPlatformColors: Record< @@ -16,7 +17,8 @@ export const EmbedlyPlatformColors: Record< [EmbedlyPlatformType.Instagram]: [225, 48, 108], [EmbedlyPlatformType.TikTok]: [57, 118, 132], [EmbedlyPlatformType.CBC]: [215, 36, 42], - [EmbedlyPlatformType.Threads]: [0, 0, 0] + [EmbedlyPlatformType.Threads]: [0, 0, 0], + [EmbedlyPlatformType.Reddit]: [255, 86, 0] }; export interface StatsData { @@ -63,5 +65,6 @@ export const emojis: Emojis = { [EmbedlyPlatformType.Instagram]: "<:instagram:1386639712013254748>", [EmbedlyPlatformType.TikTok]: "<:tiktok:1386641825963708446>", [EmbedlyPlatformType.CBC]: "<:cbc:1409997044495683674>", - [EmbedlyPlatformType.Threads]: "<:threads:1413343483929956446>" + [EmbedlyPlatformType.Threads]: "<:threads:1413343483929956446>", + [EmbedlyPlatformType.Reddit]: "<:reddit:1461320093240655922>" }; diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json index 65521da..3a138a9 100644 --- a/packages/types/tsconfig.json +++ b/packages/types/tsconfig.json @@ -1,7 +1,8 @@ { - "extends": "@embedly/config/typescript", + "extends": "@embedly/config/tsconfig.json", "compilerOptions": { - "lib": ["WebWorker", "ESNext"] + "lib": ["WebWorker", "ESNext"], + "outDir": "./dist" }, "exclude": ["node_modules"], "include": ["src"]