diff --git a/.github/workflows/build-binary.yml b/.github/workflows/build-binary.yml index c7718f2..553a7a8 100644 --- a/.github/workflows/build-binary.yml +++ b/.github/workflows/build-binary.yml @@ -2,6 +2,7 @@ name: Build binary on: workflow_dispatch: push: + branches: [main] jobs: build-binary: name: Build binary diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml index d351148..53c3e07 100644 --- a/.github/workflows/run-checks.yml +++ b/.github/workflows/run-checks.yml @@ -2,6 +2,7 @@ name: Run checks on: workflow_dispatch: push: + branches: [main] pull_request: jobs: cargo-check: diff --git a/Cargo.lock b/Cargo.lock index d497a75..2842f11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -82,9 +82,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -163,6 +163,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cssparser" version = "0.34.0" @@ -211,7 +217,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -257,8 +263,11 @@ dependencies = [ "async-trait", "dotenvy", "instagram-client", + "reddit-client", "reqwest", "rustls", + "scraper", + "thiserror", "tiktok-client", "tokio", "tracing", @@ -272,11 +281,17 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "2a7a45518d2863d18aa47f4a0cf9faec2aa4304cc09df5e41299f276b3ad135e" dependencies = [ "cfg-if", + "core_detect", + "multiversion", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -303,9 +318,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "fnv" @@ -692,9 +707,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.1" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -716,9 +731,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "itoa" @@ -787,9 +802,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -886,15 +901,43 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", "windows-sys 0.61.2", ] +[[package]] +name = "multiversion" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "target-features", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "native-tls" version = "0.2.18" @@ -1169,6 +1212,18 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +[[package]] +name = "reddit-client" +version = "0.1.0" +dependencies = [ + "reqwest", + "scraper", + "thiserror", + "tokio", + "tracing", + "url", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1258,9 +1313,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" dependencies = [ "log", "once_cell", @@ -1466,7 +1521,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1490,7 +1545,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1575,9 +1630,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" [[package]] name = "socket2" @@ -1639,9 +1694,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -1689,6 +1744,12 @@ dependencies = [ "libc", ] +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "tempfile" version = "3.27.0" @@ -1730,7 +1791,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1821,7 +1882,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1836,9 +1897,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -2181,9 +2242,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -2194,9 +2255,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -2204,9 +2265,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2214,31 +2275,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -2465,7 +2526,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2476,18 +2537,18 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 9b464d7..81234ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,10 +3,12 @@ resolver = "3" members = [ "packages/embdr", "packages/instagram-client", + "packages/reddit-client", "packages/tiktok-client", ] default-members = ["packages/embdr"] [workspace.dependencies] instagram-client = { path = "packages/instagram-client" } +reddit-client = { path = "packages/reddit-client" } tiktok-client = { path = "packages/tiktok-client" } diff --git a/packages/embdr/Cargo.toml b/packages/embdr/Cargo.toml index 94a74eb..b86920d 100644 --- a/packages/embdr/Cargo.toml +++ b/packages/embdr/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] instagram-client.workspace = true +reddit-client.workspace = true tiktok-client.workspace = true # Environment @@ -32,3 +33,5 @@ reqwest = { version = "0.12", features = ["cookies"] } # Other async-trait = "0.1.92" url = "2.5.8" +scraper = "0.22" +thiserror = "2" diff --git a/packages/embdr/src/main.rs b/packages/embdr/src/main.rs index 7b558a1..4ad1d03 100644 --- a/packages/embdr/src/main.rs +++ b/packages/embdr/src/main.rs @@ -84,13 +84,28 @@ impl EMBDR { let mut container = ContainerBuilder::new().accent_color(Some(source.color())); - container = container.component( - TextDisplayBuilder::new(format!( + let header = match (&media_data.title, &media_data.community) { + (Some(title), Some(community)) => format!( + "## [{}]({}) · [{}]({})", + community.name, community.url, + title, media_data.author.url + ), + (Some(title), None) => format!( + "## [{}]({})", + title, media_data.author.url + ), + (None, Some(community)) => format!( + "## [{}]({}) · Post by [{}]({})", + community.name, community.url, + media_data.author.name, media_data.author.url + ), + (None, None) => format!( "## Post by [{}]({})", media_data.author.name, media_data.author.url - )) - .build(), - ); + ), + }; + + container = container.component(TextDisplayBuilder::new(header).build()); if let Some(description) = media_data.description.clone() { container = @@ -123,12 +138,18 @@ impl EMBDR { .properties .iter() .map(|property| match property { + sources::MediaProperty::Score(count) => { + format!("{} {}", property.emoji(), count) + } sources::MediaProperty::LikeCount(count) => { format!("{} {}", property.emoji(), count) } sources::MediaProperty::CommentCount(count) => { format!("{} {}", property.emoji(), count) } + sources::MediaProperty::RepostCount(count) => { + format!("{} {}", property.emoji(), count) + } }) .collect::>() .join(" • "); @@ -150,6 +171,11 @@ impl EMBDR { .build() .into(), ); + + tracing::error!( + error = ?media_data_result.unwrap_err(), + "Failed to extract media from the provided link" + ); } http.update_message(message.channel_id, message.id) diff --git a/packages/embdr/src/sources/instagram.rs b/packages/embdr/src/sources/instagram.rs index 5ddd35c..b7ffbdc 100644 --- a/packages/embdr/src/sources/instagram.rs +++ b/packages/embdr/src/sources/instagram.rs @@ -66,6 +66,7 @@ impl Source for InstagramSource { Ok(MediaData { id: instagram_post.shortcode.clone(), + title: None, author: MediaAuthor { name: instagram_post.owner.username.clone(), url: format!( @@ -73,6 +74,7 @@ impl Source for InstagramSource { instagram_post.owner.username ), }, + community: None, description: instagram_post.caption.clone().into(), items: instagram_post .media_items diff --git a/packages/embdr/src/sources/mod.rs b/packages/embdr/src/sources/mod.rs index 52ec974..e331f9c 100644 --- a/packages/embdr/src/sources/mod.rs +++ b/packages/embdr/src/sources/mod.rs @@ -1,9 +1,11 @@ use url::Url; use crate::sources::instagram::InstagramSource; +use crate::sources::reddit::RedditSource; use crate::sources::tiktok::TiktokSource; mod instagram; +mod reddit; mod tiktok; #[allow(dead_code)] @@ -32,7 +34,11 @@ pub trait Source { } pub fn get_sources() -> Sources { - vec![Box::new(InstagramSource::new()), Box::new(TiktokSource::new())] + vec![ + Box::new(InstagramSource::new()), + Box::new(RedditSource::new()), + Box::new(TiktokSource::new()), + ] } pub type Sources = Vec>; @@ -40,7 +46,9 @@ pub type Sources = Vec>; #[derive(Debug)] pub struct MediaData { pub id: String, + pub title: Option, pub author: MediaAuthor, + pub community: Option, pub description: Option, pub items: Vec, pub properties: Vec, @@ -52,6 +60,12 @@ pub struct MediaAuthor { pub url: String, } +#[derive(Debug)] +pub struct MediaCommunity { + pub name: String, + pub url: String, +} + #[derive(Debug)] pub struct MediaItem { pub url: String, @@ -59,15 +73,19 @@ pub struct MediaItem { #[derive(Debug)] pub enum MediaProperty { + Score(u64), LikeCount(u64), CommentCount(u64), + RepostCount(u64), } impl MediaProperty { pub fn emoji(&self) -> String { match self { + MediaProperty::Score(_) => "↕️".to_string(), MediaProperty::LikeCount(_) => "❤️".to_string(), MediaProperty::CommentCount(_) => "💬".to_string(), + MediaProperty::RepostCount(_) => "🔁".to_string(), } } } diff --git a/packages/embdr/src/sources/reddit.rs b/packages/embdr/src/sources/reddit.rs new file mode 100644 index 0000000..3382132 --- /dev/null +++ b/packages/embdr/src/sources/reddit.rs @@ -0,0 +1,91 @@ +use url::Url; + +use reddit_client::RedditClient; + +use crate::sources::{MediaAuthor, MediaCommunity, MediaData, MediaItem, MediaProperty, Source}; + +pub struct RedditSource { + reddit_client: RedditClient, +} + +impl RedditSource { + pub fn new() -> Self { + let reddit_client = RedditClient::new().expect("Failed to create reddit client"); + Self { reddit_client } + } +} + +#[async_trait::async_trait] +impl Source for RedditSource { + fn name(&self) -> String { + "Reddit".to_string() + } + + fn id(&self) -> String { + "reddit".to_string() + } + + fn short_id(&self) -> String { + "rd".to_string() + } + + fn color(&self) -> u32 { + 0xFF4500 + } + + fn predicate(&self, url: &Url) -> bool { + let Some(hostname) = url.host_str() else { + return false; + }; + + let hostname_matches = hostname == "www.reddit.com" + || hostname == "reddit.com" + || hostname == "old.reddit.com" + || hostname == "v.redd.it"; + + let path = url.path(); + let is_reddit_path = path.contains("/comments/") || path.contains("/s/"); + + return hostname_matches && is_reddit_path; + } + + async fn extract_media( + &self, + url: &Url, + ) -> Result> { + let reddit_post = self.reddit_client.fetch_from_url(url.as_str()).await?; + + let description = reddit_post.selftext.filter(|text| !text.is_empty()); + + let community = if !reddit_post.subreddit.is_empty() { + Some(MediaCommunity { + name: reddit_post.subreddit.clone(), + url: format!("https://www.reddit.com/{}", reddit_post.subreddit), + }) + } else { + None + }; + + Ok(MediaData { + id: reddit_post.id, + title: Some(reddit_post.title.clone()), + author: MediaAuthor { + name: format!("u/{}", reddit_post.author), + url: format!("https://www.reddit.com/user/{}", reddit_post.author), + }, + community, + description, + items: reddit_post + .media_items + .into_iter() + .map(|item| MediaItem { + url: item.url().to_string(), + }) + .collect(), + properties: vec![ + MediaProperty::Score(reddit_post.score), + MediaProperty::CommentCount(reddit_post.num_comments), + ], + }) + } +} diff --git a/packages/embdr/src/sources/tiktok.rs b/packages/embdr/src/sources/tiktok.rs index f0b2761..e0a9e0f 100644 --- a/packages/embdr/src/sources/tiktok.rs +++ b/packages/embdr/src/sources/tiktok.rs @@ -3,7 +3,7 @@ use url::Url; use tiktok_client::TiktokClient; -use crate::sources::{MediaAuthor, MediaData, MediaItem, Source}; +use crate::sources::{MediaAuthor, MediaData, MediaItem, MediaProperty, Source}; pub struct TiktokSource { tiktok_client: TiktokClient, @@ -27,6 +27,93 @@ impl TiktokSource { } } +/// Parse stats from a TikTok description. +/// Format: "**❤️ 341 💬 6 🔁 59**" at the end of the text. +/// Numbers may have K/M suffixes (e.g., "203.1K"). +/// Returns (cleaned_description, likes, comments, reposts). +fn parse_tiktok_stats(content: &str) -> (Option, u64, u64, u64) { + let mut likes = 0u64; + let mut comments = 0u64; + let mut reposts = 0u64; + + let text = content.trim(); + + // Find the stats line: starts with ❤ (may be inside **bold**) + let stats_start = text.rfind("❤"); + if let Some(start) = stats_start { + let stats_line = &text[start..]; + + // Collect all segments, splitting on any whitespace (including \u2000) + let segments: Vec<&str> = stats_line.split(|c: char| c.is_whitespace()).filter(|s| !s.is_empty()).collect(); + let mut i = 0; + while i < segments.len() { + let seg = segments[i]; + let has_emoji = seg.contains('❤') || seg.contains('💬') || seg.contains('🔁'); + + if has_emoji { + // Extract digits (and optional decimal/K/M) from this segment + if let Some(num) = extract_number(seg) { + assign_stat(seg, num, &mut likes, &mut comments, &mut reposts); + } + // Also check the next segment for a standalone number + else if i + 1 < segments.len() { + if let Some(num) = extract_number(segments[i + 1]) { + assign_stat(seg, num, &mut likes, &mut comments, &mut reposts); + i += 1; + } + } + } + i += 1; + } + + // Strip the stats line and trailing bold markdown from the description + let cleaned = text[..start].trim_end_matches("**").trim().to_string(); + let description = if cleaned.is_empty() { + None + } else { + Some(cleaned) + }; + + (description, likes, comments, reposts) + } else { + let description = if text.is_empty() { + None + } else { + Some(text.to_string()) + }; + (description, 0, 0, 0) + } +} + +/// Extract a number from a segment, handling K/M suffixes. +fn extract_number(s: &str) -> Option { + let num_str: String = s.chars().filter(|c| c.is_ascii_digit() || *c == '.').collect(); + if num_str.is_empty() { + return None; + } + + let has_k = s.contains('K') || s.contains('k'); + let has_m = s.contains('M') || s.contains('m'); + + let base: f64 = num_str.parse().ok()?; + let value = if has_m { + base * 1_000_000.0 + } else if has_k { + base * 1_000.0 + } else { + base + }; + + Some(value as u64) +} + +/// Assign a parsed number to the correct stat based on which emoji is present. +fn assign_stat(seg: &str, num: u64, likes: &mut u64, comments: &mut u64, reposts: &mut u64) { + if seg.contains('❤') { *likes = num; } + else if seg.contains('💬') { *comments = num; } + else if seg.contains('🔁') { *reposts = num; } +} + #[async_trait::async_trait] impl Source for TiktokSource { fn name(&self) -> String { @@ -61,20 +148,29 @@ impl Source for TiktokSource { url: &Url, ) -> Result> { let tiktok_post = self.tiktok_client.fetch_from_url(url.as_str()).await?; - println!("TikTok post: {:#?}", tiktok_post); - let description = if tiktok_post.content.is_empty() { - None - } else { - Some(tiktok_post.content.clone()) - }; + let (description, likes, comments, reposts) = + parse_tiktok_stats(&tiktok_post.content); + + let mut properties = Vec::new(); + if likes > 0 { + properties.push(MediaProperty::LikeCount(likes)); + } + if comments > 0 { + properties.push(MediaProperty::CommentCount(comments)); + } + if reposts > 0 { + properties.push(MediaProperty::RepostCount(reposts)); + } Ok(MediaData { id: tiktok_post.id.clone(), + title: None, author: MediaAuthor { name: tiktok_post.account.display_name.clone(), url: format!("https://www.tiktok.com/@{}", tiktok_post.account.username), }, + community: None, description, items: tiktok_post .media_attachments @@ -83,7 +179,72 @@ impl Source for TiktokSource { url: item.url.clone(), }) .collect(), - properties: vec![], + properties, }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_tiktok_stats() { + let content = "**Deze Guy😭😭😭**\n#rijbewijs #rijles #rijschool \n\n❤️341 💬 6 🔁 59"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "**Deze Guy😭😭😭**\n#rijbewijs #rijles #rijschool"); + assert_eq!(likes, 341); + assert_eq!(comments, 6); + assert_eq!(reposts, 59); + } + + #[test] + fn test_parse_tiktok_stats_no_stats() { + let content = "Just a regular post"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Just a regular post"); + assert_eq!(likes, 0); + assert_eq!(comments, 0); + assert_eq!(reposts, 0); + } + + #[test] + fn test_parse_tiktok_stats_only_likes() { + let content = "Nice video ❤️ 1200"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Nice video"); + assert_eq!(likes, 1200); + assert_eq!(comments, 0); + assert_eq!(reposts, 0); + } + + #[test] + fn test_parse_tiktok_stats_trailing_bold() { + let content = "Some caption **bold text** ❤️ 100 💬 5 🔁 2"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Some caption **bold text**"); + assert_eq!(likes, 100); + assert_eq!(comments, 5); + assert_eq!(reposts, 2); + } + + #[test] + fn test_parse_tiktok_stats_k_suffix() { + let content = "**❤️\u{2000}203.1K\u{2000}💬\u{2000}718\u{2000}🔁\u{2000}4.8K**"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc, None); + assert_eq!(likes, 203_100); + assert_eq!(comments, 718); + assert_eq!(reposts, 4_800); + } + + #[test] + fn test_parse_tiktok_stats_bold_wrapped() { + let content = "Caption\n\n**❤️ 537 💬 3 🔁 13**"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Caption"); + assert_eq!(likes, 537); + assert_eq!(comments, 3); + assert_eq!(reposts, 13); + } +} diff --git a/packages/reddit-client/Cargo.toml b/packages/reddit-client/Cargo.toml new file mode 100644 index 0000000..1ce4edb --- /dev/null +++ b/packages/reddit-client/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "reddit-client" +version = "0.1.0" +edition = "2024" + +[dependencies] +reqwest = { version = "0.12", features = ["cookies"] } +scraper = "0.22" +url = "2" +thiserror = "2" +tracing = "0.1" +tokio = { version = "1", features = ["time", "macros", "rt-multi-thread"] } + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/packages/reddit-client/src/lib.rs b/packages/reddit-client/src/lib.rs new file mode 100644 index 0000000..bbde48c --- /dev/null +++ b/packages/reddit-client/src/lib.rs @@ -0,0 +1,364 @@ +use reqwest::header::{HeaderMap, USER_AGENT}; +use reqwest::{Client, StatusCode}; +use scraper::{Html, Selector}; +use thiserror::Error; + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +#[derive(Debug, Error)] +pub enum RedditError { + #[error("HTTP request failed: {0}")] + Reqwest(#[from] reqwest::Error), + + #[error("API returned no data for URL: {url}")] + NoData { url: String }, + + #[error("Unexpected response with status {status}: {body}")] + UnexpectedResponse { status: StatusCode, body: String }, + + #[error("Failed to parse page: {0}")] + Parse(String), +} + +pub type Result = std::result::Result; + +// --------------------------------------------------------------------------- +// Types — normalized / public API +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct RedditPost { + pub id: String, + pub title: String, + pub author: String, + pub subreddit: String, + pub selftext: Option, + pub permalink: String, + pub score: u64, + pub num_comments: u64, + pub media_items: Vec, +} + +#[derive(Debug, Clone)] +pub enum RedditMediaItem { + Image { url: String }, + Video { url: String }, +} + +impl RedditMediaItem { + pub fn url(&self) -> &str { + match self { + Self::Image { url } | Self::Video { url } => url, + } + } +} + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const VXREDDIT_BASE_URL: &str = "https://vxreddit.com"; + +/// User-Agent that matches vxReddit's bot detection for social-preview crawlers. +const BOT_USER_AGENT: &str = "Discordbot/2.0"; + +// --------------------------------------------------------------------------- +// URL helpers +// --------------------------------------------------------------------------- + +/// Extract the post ID from various Reddit URL formats. +pub fn extract_post_id(url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + let comments_idx = parts.iter().position(|s| *s == "comments")?; + let post_id = parts.get(comments_idx + 1)?; + Some(post_id.to_string()) +} + +fn strip_reddit_domain(url: &str) -> &str { + url.trim_start_matches("https://www.reddit.com") + .trim_start_matches("https://reddit.com") + .trim_start_matches("https://old.reddit.com") +} + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +fn bot_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, BOT_USER_AGENT.parse().unwrap()); + headers +} + +/// Reddit client that fetches post data via vxReddit's embed pages. +pub struct RedditClient { + http_client: Client, +} + +impl RedditClient { + /// Create a new client with default settings. + pub fn new() -> Result { + let http_client = Client::builder() + .default_headers(bot_headers()) + .cookie_store(true) + .build()?; + Ok(Self { http_client }) + } + + /// Create a new client with a provided `reqwest::Client`. + pub fn new_with_http(http_client: Client) -> Self { + Self { http_client } + } + + /// Fetch post data from a Reddit URL via vxReddit's embed page. + pub async fn fetch_from_url(&self, url: &str) -> Result { + let path = strip_reddit_domain(url); + let vx_url = format!("{}{}", VXREDDIT_BASE_URL, path); + + let max_retries = 3; + let mut last_status = None; + + for attempt in 0..max_retries { + let response = self.http_client.get(&vx_url).send().await?; + + let status = response.status(); + if status.is_success() { + let html = response.text().await?; + return parse_embed_page(&html, url); + } + + if status == StatusCode::TOO_MANY_REQUESTS { + last_status = Some(status); + let wait_ms = 1000 * 2u64.pow(attempt); + tracing::warn!( + "vxReddit request got 429 — retrying in {}ms (attempt {})", + wait_ms, + attempt + 1, + ); + tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await; + continue; + } + + let body = response.text().await.unwrap_or_default(); + return Err(RedditError::UnexpectedResponse { status, body }); + } + + Err(RedditError::UnexpectedResponse { + status: last_status.unwrap_or(StatusCode::TOO_MANY_REQUESTS), + body: "rate limited — retries exhausted".to_string(), + }) + } +} + +// --------------------------------------------------------------------------- +// HTML meta tag parsing +// --------------------------------------------------------------------------- + +fn parse_embed_page(html: &str, original_url: &str) -> Result { + let document = Html::parse_document(html); + + let meta_sel = Selector::parse("meta").map_err(|e| { + RedditError::Parse(format!("Invalid selector: {}", e)) + })?; + + let mut og_site_name = None; + let mut og_title = None; + let mut og_description = None; + let mut og_url = None; + let mut og_images: Vec = Vec::new(); + let mut og_video = None; + let mut oembed_title = None; + + for meta in document.select(&meta_sel) { + let prop = meta + .attr("property") + .or_else(|| meta.attr("name")) + .unwrap_or_default(); + let content = meta.attr("content").unwrap_or_default(); + + match prop { + "og:site_name" => og_site_name = Some(content.to_string()), + "og:title" => og_title = Some(content.to_string()), + "og:description" => og_description = Some(content.to_string()), + "og:url" => og_url = Some(content.to_string()), + "og:image" => og_images.push(content.to_string()), + "og:video" => og_video = Some(content.to_string()), + _ => {} + } + } + + // Extract oembed title from + let link_sel = Selector::parse("link[rel='alternate'][type='application/json+oembed']").ok(); + if let Some(sel) = link_sel { + if let Some(link) = document.select(&sel).next() { + oembed_title = link.attr("title").map(|s| s.to_string()); + } + } + + // Determine the actual post title: + // - For text/link posts where title==text: og:title is "vxReddit", real title is in oembed title. + // - For text posts with selftext: og:title has the real title. + // - For image/video posts: og:title has the real title. + let is_vxreddit_title = og_title.as_deref() == Some("vxReddit"); + + let title = if is_vxreddit_title { + oembed_title + .clone() + .or_else(|| og_description.clone()) + .unwrap_or_default() + } else { + og_title.unwrap_or_default() + }; + + // Selftext: og:description contains selftext only when it differs from the + // resolved title AND there are no media items (image/video posts put the + // title in og:description). + let has_media = !og_images.is_empty() || og_video.is_some(); + let selftext = og_description + .filter(|s| !s.is_empty()) + .filter(|desc| *desc != title) + .filter(|_| !has_media); + + // Parse og:site_name: "{author} on r/{subreddit} - ⬆️ {upvotes} | 💬 {comments}" + let (author, subreddit, score, num_comments) = + parse_stats_line(og_site_name.as_deref().unwrap_or_default()); + + // Build permalink + let permalink = og_url + .clone() + .unwrap_or_else(|| original_url.to_string()); + + // Extract post ID from permalink + let id = extract_post_id(&permalink).unwrap_or_default(); + + // Build media items. + // For video posts, og:image is just the thumbnail — skip it. + let mut media_items: Vec = Vec::new(); + + if let Some(video_url) = og_video { + media_items.push(RedditMediaItem::Video { url: video_url }); + } else { + for img_url in &og_images { + if !media_items + .iter() + .any(|item| item.url() == img_url.as_str()) + { + media_items.push(RedditMediaItem::Image { + url: img_url.clone(), + }); + } + } + } + + Ok(RedditPost { + id, + title, + author, + subreddit, + selftext, + permalink, + score, + num_comments, + media_items, + }) +} + +/// Parse the stats line from og:site_name. +/// Format: "{author} on r/{subreddit} - ⬆️ {upvotes} | 💬 {comments}" +fn parse_stats_line(line: &str) -> (String, String, u64, u64) { + let mut author = String::new(); + let mut subreddit = String::new(); + let mut score = 0u64; + let mut num_comments = 0u64; + + // Extract author: everything before " on " + if let Some(on_idx) = line.find(" on ") { + author = line[..on_idx] + .trim_start_matches("u/") + .trim_start_matches("/u/") + .to_string(); + } + + // Extract subreddit: between "on " and " - " + if let Some(on_idx) = line.find(" on ") { + let rest = &line[on_idx + 4..]; + if let Some(dash_idx) = rest.find(" - ") { + subreddit = rest[..dash_idx].to_string(); + } + } + + // Extract upvotes: between "⬆️ " and " |" or end + if let Some(up_idx) = line.find("⬆️ ") { + let rest = &line[up_idx + "⬆️ ".len()..]; + let num_str: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect(); + score = num_str.parse().unwrap_or(0); + } + + // Extract comments: after "💬 " + if let Some(comment_idx) = line.find("💬 ") { + let rest = &line[comment_idx + "💬 ".len()..]; + let num_str: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect(); + num_comments = num_str.parse().unwrap_or(0); + } + + (author, subreddit, score, num_comments) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_post_id() { + let url = "https://www.reddit.com/r/rust/comments/abc123/title_here/"; + assert_eq!(extract_post_id(url).unwrap(), "abc123"); + + let url = "https://www.reddit.com/comments/abc123/title/"; + assert_eq!(extract_post_id(url).unwrap(), "abc123"); + + let url = "https://www.reddit.com/r/programming"; + assert!(extract_post_id(url).is_none()); + } + + #[test] + fn test_parse_stats_line() { + let (author, sub, score, comments) = + parse_stats_line("u/iquizuanswer on r/Piracy - ⬆️ 1380 | 💬 34"); + assert_eq!(author, "iquizuanswer"); + assert_eq!(sub, "r/Piracy"); + assert_eq!(score, 1380); + assert_eq!(comments, 34); + } + + #[test] + fn test_parse_stats_line_no_comments() { + let (author, sub, score, comments) = + parse_stats_line("u/test on r/rust - ⬆️ 42"); + assert_eq!(author, "test"); + assert_eq!(sub, "r/rust"); + assert_eq!(score, 42); + assert_eq!(comments, 0); + } + + #[test] + fn test_strip_reddit_domain() { + assert_eq!( + strip_reddit_domain("https://www.reddit.com/r/rust/comments/abc123/"), + "/r/rust/comments/abc123/" + ); + assert_eq!( + strip_reddit_domain("https://old.reddit.com/r/rust/comments/abc123/"), + "/r/rust/comments/abc123/" + ); + assert_eq!( + strip_reddit_domain("https://reddit.com/r/rust/comments/abc123/"), + "/r/rust/comments/abc123/" + ); + } +}