From 95afe65274b93f25122221d197e2f92bce919c8b Mon Sep 17 00:00:00 2001 From: CoreByte <62032154+CoreBytee@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:13:08 +0200 Subject: [PATCH 1/4] Add Reddit source support via RSS feed parsing (wip) --- Cargo.lock | 321 ++++++++++++++++++++++ Cargo.toml | 2 + packages/embdr/Cargo.toml | 1 + packages/embdr/src/main.rs | 5 + packages/embdr/src/sources/mod.rs | 7 +- packages/embdr/src/sources/reddit.rs | 79 ++++++ packages/reddit-client/Cargo.toml | 15 ++ packages/reddit-client/src/lib.rs | 386 +++++++++++++++++++++++++++ 8 files changed, 815 insertions(+), 1 deletion(-) create mode 100644 packages/embdr/src/sources/reddit.rs create mode 100644 packages/reddit-client/Cargo.toml create mode 100644 packages/reddit-client/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 9d816a6..1b486ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,6 +68,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -157,12 +163,46 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -189,6 +229,27 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + [[package]] name = "embdr" version = "0.1.0" @@ -196,6 +257,7 @@ dependencies = [ "async-trait", "dotenvy", "instagram-client", + "reddit-client", "rustls", "tokio", "tracing", @@ -274,6 +336,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -313,6 +385,24 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -366,6 +456,18 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "html5ever" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" +dependencies = [ + "log", + "mac", + "markup5ever", + "match_token", +] + [[package]] name = "http" version = "1.5.0" @@ -738,6 +840,37 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "match_token" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "memchr" version = "2.8.3" @@ -778,6 +911,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -889,6 +1028,58 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -916,6 +1107,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -956,6 +1153,33 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "reddit-client" +version = "0.1.0" +dependencies = [ + "reqwest", + "scraper", + "thiserror", + "tokio", + "tracing", + "url", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1153,6 +1377,21 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc3d051b884f40e309de6c149734eab57aa8cc1347992710dc80bcc1c2194c15" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -1176,6 +1415,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.28" @@ -1258,6 +1516,15 @@ dependencies = [ "serde", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -1305,6 +1572,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -1333,6 +1606,31 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1415,6 +1713,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "thiserror" version = "2.0.20" @@ -1782,6 +2091,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "untrusted" version = "0.9.0" @@ -1806,6 +2121,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index a8cda64..164e655 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,8 +3,10 @@ resolver = "3" members = [ "packages/embdr", "packages/instagram-client", + "packages/reddit-client", ] default-members = ["packages/embdr"] [workspace.dependencies] instagram-client = { path = "packages/instagram-client" } +reddit-client = { path = "packages/reddit-client" } diff --git a/packages/embdr/Cargo.toml b/packages/embdr/Cargo.toml index 0f5c1e3..483711e 100644 --- a/packages/embdr/Cargo.toml +++ b/packages/embdr/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] instagram-client.workspace = true +reddit-client.workspace = true # Environment dotenvy = "0.15.7" diff --git a/packages/embdr/src/main.rs b/packages/embdr/src/main.rs index f292343..3991af5 100644 --- a/packages/embdr/src/main.rs +++ b/packages/embdr/src/main.rs @@ -150,6 +150,11 @@ impl EMBDR { .build() .into(), ); + + tracing::error!( + error = ?media_data_result.unwrap_err(), + "Failed to extract media from the provided link" + ); } let result = http diff --git a/packages/embdr/src/sources/mod.rs b/packages/embdr/src/sources/mod.rs index 9dd1921..0b02ef4 100644 --- a/packages/embdr/src/sources/mod.rs +++ b/packages/embdr/src/sources/mod.rs @@ -1,8 +1,10 @@ use url::Url; use crate::sources::instagram::InstagramSource; +use crate::sources::reddit::RedditSource; mod instagram; +mod reddit; #[async_trait::async_trait] pub trait Source { @@ -29,7 +31,10 @@ pub trait Source { } pub fn get_sources() -> Sources { - vec![Box::new(InstagramSource::new())] + vec![ + Box::new(InstagramSource::new()), + Box::new(RedditSource::new()), + ] } pub type Sources = Vec>; diff --git a/packages/embdr/src/sources/reddit.rs b/packages/embdr/src/sources/reddit.rs new file mode 100644 index 0000000..a73842e --- /dev/null +++ b/packages/embdr/src/sources/reddit.rs @@ -0,0 +1,79 @@ +use url::Url; + +use reddit_client::RedditClient; + +use crate::sources::{MediaAuthor, MediaData, MediaItem, MediaProperty, Source}; + +pub struct RedditSource { + reddit_client: RedditClient, +} + +impl RedditSource { + pub fn new() -> Self { + let reddit_client = RedditClient::new().expect("Failed to create reddit client"); + Self { reddit_client } + } +} + +#[async_trait::async_trait] +impl Source for RedditSource { + fn name(&self) -> String { + "Reddit".to_string() + } + + fn id(&self) -> String { + "reddit".to_string() + } + + fn short_id(&self) -> String { + "rd".to_string() + } + + fn color(&self) -> u32 { + 0xFF4500 + } + + fn predicate(&self, url: &Url) -> bool { + let Some(hostname) = url.host_str() else { + return false; + }; + + let hostname_matches = hostname == "www.reddit.com" + || hostname == "reddit.com" + || hostname == "old.reddit.com" + || hostname == "v.redd.it"; + + let has_comments_path = url.path().contains("/comments/"); + + return hostname_matches && has_comments_path; + } + + async fn extract_media( + &self, + url: &Url, + ) -> Result> { + let reddit_post = self.reddit_client.fetch_from_url(url.as_str()).await?; + + let description = reddit_post.selftext.filter(|text| !text.is_empty()); + + Ok(MediaData { + id: reddit_post.id, + author: MediaAuthor { + name: format!("u/{}", reddit_post.author), + url: format!("https://www.reddit.com/user/{}", reddit_post.author), + }, + description, + items: reddit_post + .media_items + .into_iter() + .map(|item| MediaItem { + url: item.url().to_string(), + }) + .collect(), + properties: vec![ + // MediaProperty::LikeCount(reddit_post.score), // not in rss data + MediaProperty::CommentCount(reddit_post.num_comments), + ], + }) + } +} diff --git a/packages/reddit-client/Cargo.toml b/packages/reddit-client/Cargo.toml new file mode 100644 index 0000000..1ce4edb --- /dev/null +++ b/packages/reddit-client/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "reddit-client" +version = "0.1.0" +edition = "2024" + +[dependencies] +reqwest = { version = "0.12", features = ["cookies"] } +scraper = "0.22" +url = "2" +thiserror = "2" +tracing = "0.1" +tokio = { version = "1", features = ["time", "macros", "rt-multi-thread"] } + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/packages/reddit-client/src/lib.rs b/packages/reddit-client/src/lib.rs new file mode 100644 index 0000000..819b8f9 --- /dev/null +++ b/packages/reddit-client/src/lib.rs @@ -0,0 +1,386 @@ +use reqwest::header::{HeaderMap, ACCEPT, USER_AGENT}; +use reqwest::{Client, StatusCode}; +use scraper::{Html, Selector}; +use thiserror::Error; + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +#[derive(Debug, Error)] +pub enum RedditError { + #[error("HTTP request failed: {0}")] + Reqwest(#[from] reqwest::Error), + + #[error("API returned no data for URL: {url}")] + NoData { url: String }, + + #[error("Unexpected response with status {status}: {body}")] + UnexpectedResponse { status: StatusCode, body: String }, + + #[error("Failed to parse RSS feed: {0}")] + Parse(String), +} + +pub type Result = std::result::Result; + +// --------------------------------------------------------------------------- +// Types — normalized / public API +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct RedditPost { + pub id: String, + pub title: String, + pub author: String, + pub selftext: Option, + pub permalink: String, + pub score: u64, + pub num_comments: u64, + pub media_items: Vec, +} + +#[derive(Debug, Clone)] +pub enum RedditMediaItem { + Image { url: String }, + Video { url: String }, +} + +impl RedditMediaItem { + pub fn url(&self) -> &str { + match self { + Self::Image { url } | Self::Video { url } => url, + } + } +} + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const DEFAULT_USER_AGENT: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; + +// --------------------------------------------------------------------------- +// URL helpers +// --------------------------------------------------------------------------- + +/// Extract the post ID from various Reddit URL formats. +pub fn extract_post_id(url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + let comments_idx = parts.iter().position(|s| *s == "comments")?; + let post_id = parts.get(comments_idx + 1)?; + Some(post_id.to_string()) +} + +/// Build the RSS feed URL for a Reddit post. +fn to_rss_url(url: &str) -> String { + let trimmed = url.trim_end_matches('/'); + format!("{}/.rss", trimmed) +} + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +fn default_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, DEFAULT_USER_AGENT.parse().unwrap()); + headers.insert(ACCEPT, "application/atom+xml".parse().unwrap()); + headers +} + +/// Reddit client that fetches post data via RSS feeds. +pub struct RedditClient { + http_client: Client, +} + +impl RedditClient { + /// Create a new client with default settings. + pub fn new() -> Result { + let client = Client::builder() + .default_headers(default_headers()) + .cookie_store(true) + .build()?; + Ok(Self { + http_client: client, + }) + } + + /// Create a new client with a provided `reqwest::Client`. + pub fn new_with_http(http_client: Client) -> Self { + Self { http_client } + } + + /// Fetch post data from a Reddit URL via its RSS feed. + pub async fn fetch_from_url(&self, url: &str) -> Result { + let rss_url = to_rss_url(url); + + let max_retries = 3; + let mut last_status = None; + + for attempt in 0..max_retries { + let response = self.http_client.get(&rss_url).send().await?; + + let status = response.status(); + if status.is_success() { + let xml = response.text().await?; + return parse_rss_feed(&xml, url); + } + + if status == StatusCode::TOO_MANY_REQUESTS { + last_status = Some(status); + let wait_ms = 1000 * 2u64.pow(attempt); + tracing::warn!( + "Reddit RSS request got 429 — retrying in {}ms (attempt {})", + wait_ms, + attempt + 1, + ); + tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await; + continue; + } + + let body = response.text().await.unwrap_or_default(); + return Err(RedditError::UnexpectedResponse { status, body }); + } + + Err(RedditError::UnexpectedResponse { + status: last_status.unwrap_or(StatusCode::TOO_MANY_REQUESTS), + body: "rate limited — retries exhausted".to_string(), + }) + } +} + +// --------------------------------------------------------------------------- +// RSS parsing +// --------------------------------------------------------------------------- + +fn parse_rss_feed(xml: &str, original_url: &str) -> Result { + let document = Html::parse_document(xml); + + // The first in the feed is the post itself; subsequent entries are comments. + let entry_selector = + Selector::parse("entry").map_err(|e| RedditError::Parse(format!("Invalid selector: {}", e)))?; + + let post_entry = document + .select(&entry_selector) + .next() + .ok_or_else(|| RedditError::NoData { + url: original_url.to_string(), + })?; + + // Extract post ID from tag (format: t3_xxxxx) + let id = get_text_content(&document, &post_entry, "id") + .map(|s| s.trim_start_matches("t3_").to_string()) + .unwrap_or_default(); + + // Extract title + let title = get_text_content(&document, &post_entry, "title").unwrap_or_default(); + + // Extract author name from + let author = { + let author_name_sel = Selector::parse("author name") + .map_err(|e| RedditError::Parse(format!("Invalid selector: {}", e)))?; + post_entry + .select(&author_name_sel) + .next() + .map(|el| el.text().collect::()) + .unwrap_or_default() + .trim_start_matches("/u/") + .trim_start_matches("u/") + .to_string() + }; + + // Extract permalink from + let permalink = post_entry + .attr("href") + .or_else(|| { + let link_sel = Selector::parse("link[rel='alternate']").ok()?; + post_entry.select(&link_sel).next()?.attr("href") + }) + .map(|s| s.to_string()) + .unwrap_or_default(); + + // Extract media from the HTML content of the first entry + let media_items = extract_media_from_entry(&post_entry); + + // Extract selftext from the content HTML + let selftext = extract_selftext_from_entry(&post_entry); + + // Count comments (entries after the first one) + let num_comments = document.select(&entry_selector).count().saturating_sub(1) as u64; + + Ok(RedditPost { + id, + title, + author, + selftext, + permalink, + score: 0, // RSS doesn't include score + num_comments, + media_items, + }) +} + +fn get_text_content( + _document: &Html, + element: &scraper::ElementRef, + tag: &str, +) -> Option { + let sel = Selector::parse(tag).ok()?; + let el = element.select(&sel).next()?; + Some(el.text().collect::()) +} + +fn extract_selftext_from_entry(entry: &scraper::ElementRef) -> Option { + let content_sel = Selector::parse("content").ok()?; + let content_el = entry.select(&content_sel).next()?; + let html_content = content_el.text().collect::(); + + // Parse the HTML content to extract text + let fragment = Html::parse_fragment(&html_content); + let md_sel = Selector::parse("div.md p").ok(); + if let Some(sel) = md_sel { + let text: String = fragment + .select(&sel) + .map(|p| p.text().collect::()) + .collect::>() + .join("\n\n"); + if !text.is_empty() { + return Some(text); + } + } + + // Fallback: extract text from any paragraph + let p_sel = Selector::parse("p").ok()?; + let text: String = fragment + .select(&p_sel) + .map(|p| p.text().collect::()) + .collect::>() + .join("\n\n"); + if text.is_empty() { + None + } else { + Some(text) + } +} + +fn extract_media_from_entry(entry: &scraper::ElementRef) -> Vec { + let mut items = Vec::new(); + + let content_sel = Selector::parse("content").ok(); + if let Some(sel) = content_sel { + if let Some(content_el) = entry.select(&sel).next() { + let html_content = content_el.text().collect::(); + let fragment = Html::parse_fragment(&html_content); + + // 1) Check for video sources first — return immediately if found + let video_sel = Selector::parse("video source, video").ok(); + if let Some(sel) = video_sel { + for el in fragment.select(&sel) { + if let Some(src) = el.attr("src") { + if src.contains("v.redd.it") || src.contains("video") { + let cleaned = src.replace("&", "&"); + items.push(RedditMediaItem::Video { url: cleaned }); + return items; + } + } + } + } + + // 2) Look for direct links (full resolution) + let link_sel = Selector::parse("a[href]").ok(); + if let Some(sel) = link_sel { + for el in fragment.select(&sel) { + if let Some(href) = el.attr("href") { + let cleaned = href.replace("&", "&"); + if cleaned.contains("v.redd.it") && cleaned.contains("DASH_") { + if !items.iter().any(|item| item.url() == &cleaned) { + items.push(RedditMediaItem::Video { url: cleaned }); + } + } else if (cleaned.contains("i.redd.it") + || cleaned.contains("i.imgur.com")) + && !items.iter().any(|item| item.url() == &cleaned) + { + items.push(RedditMediaItem::Image { url: cleaned }); + } + } + } + } + + // 3) Only add preview.redd.it if we have no i.redd.it images yet + let has_full_res = items.iter().any(|item| match item { + RedditMediaItem::Image { url } => url.contains("i.redd.it") || url.contains("i.imgur.com"), + _ => false, + }); + if !has_full_res { + let img_sel = Selector::parse("img").ok(); + if let Some(sel) = img_sel { + for el in fragment.select(&sel) { + if let Some(src) = el.attr("src") { + let cleaned = src.replace("&", "&"); + if (cleaned.contains("i.redd.it") + || cleaned.contains("preview.redd.it") + || cleaned.contains("i.imgur.com")) + && !items.iter().any(|item| item.url() == &cleaned) + { + items.push(RedditMediaItem::Image { url: cleaned }); + } + } + } + } + } + } + } + + // 4) Fallback: only if nothing else found + if items.is_empty() { + let thumb_sel = Selector::parse("media|thumbnail, thumbnail").ok(); + if let Some(sel) = thumb_sel { + for el in entry.select(&sel) { + if let Some(url) = el.attr("url") { + let cleaned = url.replace("&", "&"); + items.push(RedditMediaItem::Image { url: cleaned }); + } + } + } + } + + items +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_post_id() { + let url = "https://www.reddit.com/r/rust/comments/abc123/title_here/"; + assert_eq!(extract_post_id(url).unwrap(), "abc123"); + + let url = "https://www.reddit.com/comments/abc123/title/"; + assert_eq!(extract_post_id(url).unwrap(), "abc123"); + + let url = "https://www.reddit.com/r/programming"; + assert!(extract_post_id(url).is_none()); + } + + #[test] + fn test_to_rss_url() { + let url = "https://www.reddit.com/r/rust/comments/abc123/title/"; + assert_eq!( + to_rss_url(url), + "https://www.reddit.com/r/rust/comments/abc123/title/.rss" + ); + + let url = "https://old.reddit.com/r/rust/comments/abc123/title"; + assert_eq!( + to_rss_url(url), + "https://old.reddit.com/r/rust/comments/abc123/title/.rss" + ); + } +} From 3c67953b42e1124376fd8ab464fb8e62a4733591 Mon Sep 17 00:00:00 2001 From: CoreByte <62032154+CoreBytee@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:29:46 +0200 Subject: [PATCH 2/4] Switch to vxReddit embed pages and add community info to embeds --- packages/embdr/src/main.rs | 28 +- packages/embdr/src/sources/instagram.rs | 2 + packages/embdr/src/sources/mod.rs | 10 + packages/embdr/src/sources/reddit.rs | 20 +- packages/reddit-client/src/lib.rs | 378 +++++++++++------------- 5 files changed, 229 insertions(+), 209 deletions(-) diff --git a/packages/embdr/src/main.rs b/packages/embdr/src/main.rs index 3991af5..8f690ef 100644 --- a/packages/embdr/src/main.rs +++ b/packages/embdr/src/main.rs @@ -84,13 +84,28 @@ impl EMBDR { let mut container = ContainerBuilder::new().accent_color(Some(source.color())); - container = container.component( - TextDisplayBuilder::new(format!( + let header = match (&media_data.title, &media_data.community) { + (Some(title), Some(community)) => format!( + "## [{}]({}) · [{}]({})", + community.name, community.url, + title, media_data.author.url + ), + (Some(title), None) => format!( + "## [{}]({})", + title, media_data.author.url + ), + (None, Some(community)) => format!( + "## [{}]({}) · Post by [{}]({})", + community.name, community.url, + media_data.author.name, media_data.author.url + ), + (None, None) => format!( "## Post by [{}]({})", media_data.author.name, media_data.author.url - )) - .build(), - ); + ), + }; + + container = container.component(TextDisplayBuilder::new(header).build()); if let Some(description) = media_data.description.clone() { container = @@ -123,6 +138,9 @@ impl EMBDR { .properties .iter() .map(|property| match property { + sources::MediaProperty::Score(count) => { + format!("{} {}", property.emoji(), count) + } sources::MediaProperty::LikeCount(count) => { format!("{} {}", property.emoji(), count) } diff --git a/packages/embdr/src/sources/instagram.rs b/packages/embdr/src/sources/instagram.rs index be95a79..58318f1 100644 --- a/packages/embdr/src/sources/instagram.rs +++ b/packages/embdr/src/sources/instagram.rs @@ -54,6 +54,7 @@ impl Source for InstagramSource { Ok(MediaData { id: instagram_post.shortcode.clone(), + title: None, author: MediaAuthor { name: instagram_post.owner.username.clone(), url: format!( @@ -61,6 +62,7 @@ impl Source for InstagramSource { instagram_post.owner.username ), }, + community: None, description: instagram_post.caption.clone().into(), items: instagram_post .media_items diff --git a/packages/embdr/src/sources/mod.rs b/packages/embdr/src/sources/mod.rs index 0b02ef4..61fb242 100644 --- a/packages/embdr/src/sources/mod.rs +++ b/packages/embdr/src/sources/mod.rs @@ -42,7 +42,9 @@ pub type Sources = Vec>; #[derive(Debug)] pub struct MediaData { pub id: String, + pub title: Option, pub author: MediaAuthor, + pub community: Option, pub description: Option, pub items: Vec, pub properties: Vec, @@ -54,6 +56,12 @@ pub struct MediaAuthor { pub url: String, } +#[derive(Debug)] +pub struct MediaCommunity { + pub name: String, + pub url: String, +} + #[derive(Debug)] pub struct MediaItem { pub url: String, @@ -61,6 +69,7 @@ pub struct MediaItem { #[derive(Debug)] pub enum MediaProperty { + Score(u64), LikeCount(u64), CommentCount(u64), } @@ -68,6 +77,7 @@ pub enum MediaProperty { impl MediaProperty { pub fn emoji(&self) -> String { match self { + MediaProperty::Score(_) => "↕️".to_string(), MediaProperty::LikeCount(_) => "❤️".to_string(), MediaProperty::CommentCount(_) => "💬".to_string(), } diff --git a/packages/embdr/src/sources/reddit.rs b/packages/embdr/src/sources/reddit.rs index a73842e..3382132 100644 --- a/packages/embdr/src/sources/reddit.rs +++ b/packages/embdr/src/sources/reddit.rs @@ -2,7 +2,7 @@ use url::Url; use reddit_client::RedditClient; -use crate::sources::{MediaAuthor, MediaData, MediaItem, MediaProperty, Source}; +use crate::sources::{MediaAuthor, MediaCommunity, MediaData, MediaItem, MediaProperty, Source}; pub struct RedditSource { reddit_client: RedditClient, @@ -43,9 +43,10 @@ impl Source for RedditSource { || hostname == "old.reddit.com" || hostname == "v.redd.it"; - let has_comments_path = url.path().contains("/comments/"); + let path = url.path(); + let is_reddit_path = path.contains("/comments/") || path.contains("/s/"); - return hostname_matches && has_comments_path; + return hostname_matches && is_reddit_path; } async fn extract_media( @@ -56,12 +57,23 @@ impl Source for RedditSource { let description = reddit_post.selftext.filter(|text| !text.is_empty()); + let community = if !reddit_post.subreddit.is_empty() { + Some(MediaCommunity { + name: reddit_post.subreddit.clone(), + url: format!("https://www.reddit.com/{}", reddit_post.subreddit), + }) + } else { + None + }; + Ok(MediaData { id: reddit_post.id, + title: Some(reddit_post.title.clone()), author: MediaAuthor { name: format!("u/{}", reddit_post.author), url: format!("https://www.reddit.com/user/{}", reddit_post.author), }, + community, description, items: reddit_post .media_items @@ -71,7 +83,7 @@ impl Source for RedditSource { }) .collect(), properties: vec![ - // MediaProperty::LikeCount(reddit_post.score), // not in rss data + MediaProperty::Score(reddit_post.score), MediaProperty::CommentCount(reddit_post.num_comments), ], }) diff --git a/packages/reddit-client/src/lib.rs b/packages/reddit-client/src/lib.rs index 819b8f9..bbde48c 100644 --- a/packages/reddit-client/src/lib.rs +++ b/packages/reddit-client/src/lib.rs @@ -1,4 +1,4 @@ -use reqwest::header::{HeaderMap, ACCEPT, USER_AGENT}; +use reqwest::header::{HeaderMap, USER_AGENT}; use reqwest::{Client, StatusCode}; use scraper::{Html, Selector}; use thiserror::Error; @@ -18,7 +18,7 @@ pub enum RedditError { #[error("Unexpected response with status {status}: {body}")] UnexpectedResponse { status: StatusCode, body: String }, - #[error("Failed to parse RSS feed: {0}")] + #[error("Failed to parse page: {0}")] Parse(String), } @@ -33,6 +33,7 @@ pub struct RedditPost { pub id: String, pub title: String, pub author: String, + pub subreddit: String, pub selftext: Option, pub permalink: String, pub score: u64, @@ -58,8 +59,10 @@ impl RedditMediaItem { // Constants // --------------------------------------------------------------------------- -const DEFAULT_USER_AGENT: &str = - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; +const VXREDDIT_BASE_URL: &str = "https://vxreddit.com"; + +/// User-Agent that matches vxReddit's bot detection for social-preview crawlers. +const BOT_USER_AGENT: &str = "Discordbot/2.0"; // --------------------------------------------------------------------------- // URL helpers @@ -73,24 +76,23 @@ pub fn extract_post_id(url: &str) -> Option { Some(post_id.to_string()) } -/// Build the RSS feed URL for a Reddit post. -fn to_rss_url(url: &str) -> String { - let trimmed = url.trim_end_matches('/'); - format!("{}/.rss", trimmed) +fn strip_reddit_domain(url: &str) -> &str { + url.trim_start_matches("https://www.reddit.com") + .trim_start_matches("https://reddit.com") + .trim_start_matches("https://old.reddit.com") } // --------------------------------------------------------------------------- // Client // --------------------------------------------------------------------------- -fn default_headers() -> HeaderMap { +fn bot_headers() -> HeaderMap { let mut headers = HeaderMap::new(); - headers.insert(USER_AGENT, DEFAULT_USER_AGENT.parse().unwrap()); - headers.insert(ACCEPT, "application/atom+xml".parse().unwrap()); + headers.insert(USER_AGENT, BOT_USER_AGENT.parse().unwrap()); headers } -/// Reddit client that fetches post data via RSS feeds. +/// Reddit client that fetches post data via vxReddit's embed pages. pub struct RedditClient { http_client: Client, } @@ -98,13 +100,11 @@ pub struct RedditClient { impl RedditClient { /// Create a new client with default settings. pub fn new() -> Result { - let client = Client::builder() - .default_headers(default_headers()) + let http_client = Client::builder() + .default_headers(bot_headers()) .cookie_store(true) .build()?; - Ok(Self { - http_client: client, - }) + Ok(Self { http_client }) } /// Create a new client with a provided `reqwest::Client`. @@ -112,27 +112,28 @@ impl RedditClient { Self { http_client } } - /// Fetch post data from a Reddit URL via its RSS feed. + /// Fetch post data from a Reddit URL via vxReddit's embed page. pub async fn fetch_from_url(&self, url: &str) -> Result { - let rss_url = to_rss_url(url); + let path = strip_reddit_domain(url); + let vx_url = format!("{}{}", VXREDDIT_BASE_URL, path); let max_retries = 3; let mut last_status = None; for attempt in 0..max_retries { - let response = self.http_client.get(&rss_url).send().await?; + let response = self.http_client.get(&vx_url).send().await?; let status = response.status(); if status.is_success() { - let xml = response.text().await?; - return parse_rss_feed(&xml, url); + let html = response.text().await?; + return parse_embed_page(&html, url); } if status == StatusCode::TOO_MANY_REQUESTS { last_status = Some(status); let wait_ms = 1000 * 2u64.pow(attempt); tracing::warn!( - "Reddit RSS request got 429 — retrying in {}ms (attempt {})", + "vxReddit request got 429 — retrying in {}ms (attempt {})", wait_ms, attempt + 1, ); @@ -152,201 +153,157 @@ impl RedditClient { } // --------------------------------------------------------------------------- -// RSS parsing +// HTML meta tag parsing // --------------------------------------------------------------------------- -fn parse_rss_feed(xml: &str, original_url: &str) -> Result { - let document = Html::parse_document(xml); - - // The first in the feed is the post itself; subsequent entries are comments. - let entry_selector = - Selector::parse("entry").map_err(|e| RedditError::Parse(format!("Invalid selector: {}", e)))?; - - let post_entry = document - .select(&entry_selector) - .next() - .ok_or_else(|| RedditError::NoData { - url: original_url.to_string(), - })?; - - // Extract post ID from tag (format: t3_xxxxx) - let id = get_text_content(&document, &post_entry, "id") - .map(|s| s.trim_start_matches("t3_").to_string()) - .unwrap_or_default(); - - // Extract title - let title = get_text_content(&document, &post_entry, "title").unwrap_or_default(); - - // Extract author name from - let author = { - let author_name_sel = Selector::parse("author name") - .map_err(|e| RedditError::Parse(format!("Invalid selector: {}", e)))?; - post_entry - .select(&author_name_sel) - .next() - .map(|el| el.text().collect::()) - .unwrap_or_default() - .trim_start_matches("/u/") - .trim_start_matches("u/") - .to_string() - }; +fn parse_embed_page(html: &str, original_url: &str) -> Result { + let document = Html::parse_document(html); + + let meta_sel = Selector::parse("meta").map_err(|e| { + RedditError::Parse(format!("Invalid selector: {}", e)) + })?; + + let mut og_site_name = None; + let mut og_title = None; + let mut og_description = None; + let mut og_url = None; + let mut og_images: Vec = Vec::new(); + let mut og_video = None; + let mut oembed_title = None; + + for meta in document.select(&meta_sel) { + let prop = meta + .attr("property") + .or_else(|| meta.attr("name")) + .unwrap_or_default(); + let content = meta.attr("content").unwrap_or_default(); + + match prop { + "og:site_name" => og_site_name = Some(content.to_string()), + "og:title" => og_title = Some(content.to_string()), + "og:description" => og_description = Some(content.to_string()), + "og:url" => og_url = Some(content.to_string()), + "og:image" => og_images.push(content.to_string()), + "og:video" => og_video = Some(content.to_string()), + _ => {} + } + } - // Extract permalink from - let permalink = post_entry - .attr("href") - .or_else(|| { - let link_sel = Selector::parse("link[rel='alternate']").ok()?; - post_entry.select(&link_sel).next()?.attr("href") - }) - .map(|s| s.to_string()) - .unwrap_or_default(); + // Extract oembed title from + let link_sel = Selector::parse("link[rel='alternate'][type='application/json+oembed']").ok(); + if let Some(sel) = link_sel { + if let Some(link) = document.select(&sel).next() { + oembed_title = link.attr("title").map(|s| s.to_string()); + } + } - // Extract media from the HTML content of the first entry - let media_items = extract_media_from_entry(&post_entry); + // Determine the actual post title: + // - For text/link posts where title==text: og:title is "vxReddit", real title is in oembed title. + // - For text posts with selftext: og:title has the real title. + // - For image/video posts: og:title has the real title. + let is_vxreddit_title = og_title.as_deref() == Some("vxReddit"); - // Extract selftext from the content HTML - let selftext = extract_selftext_from_entry(&post_entry); + let title = if is_vxreddit_title { + oembed_title + .clone() + .or_else(|| og_description.clone()) + .unwrap_or_default() + } else { + og_title.unwrap_or_default() + }; - // Count comments (entries after the first one) - let num_comments = document.select(&entry_selector).count().saturating_sub(1) as u64; + // Selftext: og:description contains selftext only when it differs from the + // resolved title AND there are no media items (image/video posts put the + // title in og:description). + let has_media = !og_images.is_empty() || og_video.is_some(); + let selftext = og_description + .filter(|s| !s.is_empty()) + .filter(|desc| *desc != title) + .filter(|_| !has_media); + + // Parse og:site_name: "{author} on r/{subreddit} - ⬆️ {upvotes} | 💬 {comments}" + let (author, subreddit, score, num_comments) = + parse_stats_line(og_site_name.as_deref().unwrap_or_default()); + + // Build permalink + let permalink = og_url + .clone() + .unwrap_or_else(|| original_url.to_string()); + + // Extract post ID from permalink + let id = extract_post_id(&permalink).unwrap_or_default(); + + // Build media items. + // For video posts, og:image is just the thumbnail — skip it. + let mut media_items: Vec = Vec::new(); + + if let Some(video_url) = og_video { + media_items.push(RedditMediaItem::Video { url: video_url }); + } else { + for img_url in &og_images { + if !media_items + .iter() + .any(|item| item.url() == img_url.as_str()) + { + media_items.push(RedditMediaItem::Image { + url: img_url.clone(), + }); + } + } + } Ok(RedditPost { id, title, author, + subreddit, selftext, permalink, - score: 0, // RSS doesn't include score + score, num_comments, media_items, }) } -fn get_text_content( - _document: &Html, - element: &scraper::ElementRef, - tag: &str, -) -> Option { - let sel = Selector::parse(tag).ok()?; - let el = element.select(&sel).next()?; - Some(el.text().collect::()) -} - -fn extract_selftext_from_entry(entry: &scraper::ElementRef) -> Option { - let content_sel = Selector::parse("content").ok()?; - let content_el = entry.select(&content_sel).next()?; - let html_content = content_el.text().collect::(); - - // Parse the HTML content to extract text - let fragment = Html::parse_fragment(&html_content); - let md_sel = Selector::parse("div.md p").ok(); - if let Some(sel) = md_sel { - let text: String = fragment - .select(&sel) - .map(|p| p.text().collect::()) - .collect::>() - .join("\n\n"); - if !text.is_empty() { - return Some(text); - } +/// Parse the stats line from og:site_name. +/// Format: "{author} on r/{subreddit} - ⬆️ {upvotes} | 💬 {comments}" +fn parse_stats_line(line: &str) -> (String, String, u64, u64) { + let mut author = String::new(); + let mut subreddit = String::new(); + let mut score = 0u64; + let mut num_comments = 0u64; + + // Extract author: everything before " on " + if let Some(on_idx) = line.find(" on ") { + author = line[..on_idx] + .trim_start_matches("u/") + .trim_start_matches("/u/") + .to_string(); } - // Fallback: extract text from any paragraph - let p_sel = Selector::parse("p").ok()?; - let text: String = fragment - .select(&p_sel) - .map(|p| p.text().collect::()) - .collect::>() - .join("\n\n"); - if text.is_empty() { - None - } else { - Some(text) + // Extract subreddit: between "on " and " - " + if let Some(on_idx) = line.find(" on ") { + let rest = &line[on_idx + 4..]; + if let Some(dash_idx) = rest.find(" - ") { + subreddit = rest[..dash_idx].to_string(); + } } -} - -fn extract_media_from_entry(entry: &scraper::ElementRef) -> Vec { - let mut items = Vec::new(); - - let content_sel = Selector::parse("content").ok(); - if let Some(sel) = content_sel { - if let Some(content_el) = entry.select(&sel).next() { - let html_content = content_el.text().collect::(); - let fragment = Html::parse_fragment(&html_content); - - // 1) Check for video sources first — return immediately if found - let video_sel = Selector::parse("video source, video").ok(); - if let Some(sel) = video_sel { - for el in fragment.select(&sel) { - if let Some(src) = el.attr("src") { - if src.contains("v.redd.it") || src.contains("video") { - let cleaned = src.replace("&", "&"); - items.push(RedditMediaItem::Video { url: cleaned }); - return items; - } - } - } - } - // 2) Look for direct links (full resolution) - let link_sel = Selector::parse("a[href]").ok(); - if let Some(sel) = link_sel { - for el in fragment.select(&sel) { - if let Some(href) = el.attr("href") { - let cleaned = href.replace("&", "&"); - if cleaned.contains("v.redd.it") && cleaned.contains("DASH_") { - if !items.iter().any(|item| item.url() == &cleaned) { - items.push(RedditMediaItem::Video { url: cleaned }); - } - } else if (cleaned.contains("i.redd.it") - || cleaned.contains("i.imgur.com")) - && !items.iter().any(|item| item.url() == &cleaned) - { - items.push(RedditMediaItem::Image { url: cleaned }); - } - } - } - } - - // 3) Only add preview.redd.it if we have no i.redd.it images yet - let has_full_res = items.iter().any(|item| match item { - RedditMediaItem::Image { url } => url.contains("i.redd.it") || url.contains("i.imgur.com"), - _ => false, - }); - if !has_full_res { - let img_sel = Selector::parse("img").ok(); - if let Some(sel) = img_sel { - for el in fragment.select(&sel) { - if let Some(src) = el.attr("src") { - let cleaned = src.replace("&", "&"); - if (cleaned.contains("i.redd.it") - || cleaned.contains("preview.redd.it") - || cleaned.contains("i.imgur.com")) - && !items.iter().any(|item| item.url() == &cleaned) - { - items.push(RedditMediaItem::Image { url: cleaned }); - } - } - } - } - } - } + // Extract upvotes: between "⬆️ " and " |" or end + if let Some(up_idx) = line.find("⬆️ ") { + let rest = &line[up_idx + "⬆️ ".len()..]; + let num_str: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect(); + score = num_str.parse().unwrap_or(0); } - // 4) Fallback: only if nothing else found - if items.is_empty() { - let thumb_sel = Selector::parse("media|thumbnail, thumbnail").ok(); - if let Some(sel) = thumb_sel { - for el in entry.select(&sel) { - if let Some(url) = el.attr("url") { - let cleaned = url.replace("&", "&"); - items.push(RedditMediaItem::Image { url: cleaned }); - } - } - } + // Extract comments: after "💬 " + if let Some(comment_idx) = line.find("💬 ") { + let rest = &line[comment_idx + "💬 ".len()..]; + let num_str: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect(); + num_comments = num_str.parse().unwrap_or(0); } - items + (author, subreddit, score, num_comments) } // --------------------------------------------------------------------------- @@ -370,17 +327,38 @@ mod tests { } #[test] - fn test_to_rss_url() { - let url = "https://www.reddit.com/r/rust/comments/abc123/title/"; + fn test_parse_stats_line() { + let (author, sub, score, comments) = + parse_stats_line("u/iquizuanswer on r/Piracy - ⬆️ 1380 | 💬 34"); + assert_eq!(author, "iquizuanswer"); + assert_eq!(sub, "r/Piracy"); + assert_eq!(score, 1380); + assert_eq!(comments, 34); + } + + #[test] + fn test_parse_stats_line_no_comments() { + let (author, sub, score, comments) = + parse_stats_line("u/test on r/rust - ⬆️ 42"); + assert_eq!(author, "test"); + assert_eq!(sub, "r/rust"); + assert_eq!(score, 42); + assert_eq!(comments, 0); + } + + #[test] + fn test_strip_reddit_domain() { assert_eq!( - to_rss_url(url), - "https://www.reddit.com/r/rust/comments/abc123/title/.rss" + strip_reddit_domain("https://www.reddit.com/r/rust/comments/abc123/"), + "/r/rust/comments/abc123/" + ); + assert_eq!( + strip_reddit_domain("https://old.reddit.com/r/rust/comments/abc123/"), + "/r/rust/comments/abc123/" ); - - let url = "https://old.reddit.com/r/rust/comments/abc123/title"; assert_eq!( - to_rss_url(url), - "https://old.reddit.com/r/rust/comments/abc123/title/.rss" + strip_reddit_domain("https://reddit.com/r/rust/comments/abc123/"), + "/r/rust/comments/abc123/" ); } } From e9cefbf96763c7ced0fc703d9e412375c4a8019b Mon Sep 17 00:00:00 2001 From: CoreByte <62032154+CoreBytee@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:41:49 +0200 Subject: [PATCH 3/4] Add repost count parsing for TikTok posts --- packages/embdr/src/main.rs | 3 + packages/embdr/src/sources/mod.rs | 2 + packages/embdr/src/sources/tiktok.rs | 175 +++++++++++++++++++++++++-- 3 files changed, 172 insertions(+), 8 deletions(-) diff --git a/packages/embdr/src/main.rs b/packages/embdr/src/main.rs index 491fd4f..4ad1d03 100644 --- a/packages/embdr/src/main.rs +++ b/packages/embdr/src/main.rs @@ -147,6 +147,9 @@ impl EMBDR { sources::MediaProperty::CommentCount(count) => { format!("{} {}", property.emoji(), count) } + sources::MediaProperty::RepostCount(count) => { + format!("{} {}", property.emoji(), count) + } }) .collect::>() .join(" • "); diff --git a/packages/embdr/src/sources/mod.rs b/packages/embdr/src/sources/mod.rs index b30844b..e331f9c 100644 --- a/packages/embdr/src/sources/mod.rs +++ b/packages/embdr/src/sources/mod.rs @@ -76,6 +76,7 @@ pub enum MediaProperty { Score(u64), LikeCount(u64), CommentCount(u64), + RepostCount(u64), } impl MediaProperty { @@ -84,6 +85,7 @@ impl MediaProperty { MediaProperty::Score(_) => "↕️".to_string(), MediaProperty::LikeCount(_) => "❤️".to_string(), MediaProperty::CommentCount(_) => "💬".to_string(), + MediaProperty::RepostCount(_) => "🔁".to_string(), } } } diff --git a/packages/embdr/src/sources/tiktok.rs b/packages/embdr/src/sources/tiktok.rs index b8dabd1..e0a9e0f 100644 --- a/packages/embdr/src/sources/tiktok.rs +++ b/packages/embdr/src/sources/tiktok.rs @@ -3,7 +3,7 @@ use url::Url; use tiktok_client::TiktokClient; -use crate::sources::{MediaAuthor, MediaData, MediaItem, Source}; +use crate::sources::{MediaAuthor, MediaData, MediaItem, MediaProperty, Source}; pub struct TiktokSource { tiktok_client: TiktokClient, @@ -27,6 +27,93 @@ impl TiktokSource { } } +/// Parse stats from a TikTok description. +/// Format: "**❤️ 341 💬 6 🔁 59**" at the end of the text. +/// Numbers may have K/M suffixes (e.g., "203.1K"). +/// Returns (cleaned_description, likes, comments, reposts). +fn parse_tiktok_stats(content: &str) -> (Option, u64, u64, u64) { + let mut likes = 0u64; + let mut comments = 0u64; + let mut reposts = 0u64; + + let text = content.trim(); + + // Find the stats line: starts with ❤ (may be inside **bold**) + let stats_start = text.rfind("❤"); + if let Some(start) = stats_start { + let stats_line = &text[start..]; + + // Collect all segments, splitting on any whitespace (including \u2000) + let segments: Vec<&str> = stats_line.split(|c: char| c.is_whitespace()).filter(|s| !s.is_empty()).collect(); + let mut i = 0; + while i < segments.len() { + let seg = segments[i]; + let has_emoji = seg.contains('❤') || seg.contains('💬') || seg.contains('🔁'); + + if has_emoji { + // Extract digits (and optional decimal/K/M) from this segment + if let Some(num) = extract_number(seg) { + assign_stat(seg, num, &mut likes, &mut comments, &mut reposts); + } + // Also check the next segment for a standalone number + else if i + 1 < segments.len() { + if let Some(num) = extract_number(segments[i + 1]) { + assign_stat(seg, num, &mut likes, &mut comments, &mut reposts); + i += 1; + } + } + } + i += 1; + } + + // Strip the stats line and trailing bold markdown from the description + let cleaned = text[..start].trim_end_matches("**").trim().to_string(); + let description = if cleaned.is_empty() { + None + } else { + Some(cleaned) + }; + + (description, likes, comments, reposts) + } else { + let description = if text.is_empty() { + None + } else { + Some(text.to_string()) + }; + (description, 0, 0, 0) + } +} + +/// Extract a number from a segment, handling K/M suffixes. +fn extract_number(s: &str) -> Option { + let num_str: String = s.chars().filter(|c| c.is_ascii_digit() || *c == '.').collect(); + if num_str.is_empty() { + return None; + } + + let has_k = s.contains('K') || s.contains('k'); + let has_m = s.contains('M') || s.contains('m'); + + let base: f64 = num_str.parse().ok()?; + let value = if has_m { + base * 1_000_000.0 + } else if has_k { + base * 1_000.0 + } else { + base + }; + + Some(value as u64) +} + +/// Assign a parsed number to the correct stat based on which emoji is present. +fn assign_stat(seg: &str, num: u64, likes: &mut u64, comments: &mut u64, reposts: &mut u64) { + if seg.contains('❤') { *likes = num; } + else if seg.contains('💬') { *comments = num; } + else if seg.contains('🔁') { *reposts = num; } +} + #[async_trait::async_trait] impl Source for TiktokSource { fn name(&self) -> String { @@ -61,13 +148,20 @@ impl Source for TiktokSource { url: &Url, ) -> Result> { let tiktok_post = self.tiktok_client.fetch_from_url(url.as_str()).await?; - println!("TikTok post: {:#?}", tiktok_post); - let description = if tiktok_post.content.is_empty() { - None - } else { - Some(tiktok_post.content.clone()) - }; + let (description, likes, comments, reposts) = + parse_tiktok_stats(&tiktok_post.content); + + let mut properties = Vec::new(); + if likes > 0 { + properties.push(MediaProperty::LikeCount(likes)); + } + if comments > 0 { + properties.push(MediaProperty::CommentCount(comments)); + } + if reposts > 0 { + properties.push(MediaProperty::RepostCount(reposts)); + } Ok(MediaData { id: tiktok_post.id.clone(), @@ -85,7 +179,72 @@ impl Source for TiktokSource { url: item.url.clone(), }) .collect(), - properties: vec![], + properties, }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_tiktok_stats() { + let content = "**Deze Guy😭😭😭**\n#rijbewijs #rijles #rijschool \n\n❤️341 💬 6 🔁 59"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "**Deze Guy😭😭😭**\n#rijbewijs #rijles #rijschool"); + assert_eq!(likes, 341); + assert_eq!(comments, 6); + assert_eq!(reposts, 59); + } + + #[test] + fn test_parse_tiktok_stats_no_stats() { + let content = "Just a regular post"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Just a regular post"); + assert_eq!(likes, 0); + assert_eq!(comments, 0); + assert_eq!(reposts, 0); + } + + #[test] + fn test_parse_tiktok_stats_only_likes() { + let content = "Nice video ❤️ 1200"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Nice video"); + assert_eq!(likes, 1200); + assert_eq!(comments, 0); + assert_eq!(reposts, 0); + } + + #[test] + fn test_parse_tiktok_stats_trailing_bold() { + let content = "Some caption **bold text** ❤️ 100 💬 5 🔁 2"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Some caption **bold text**"); + assert_eq!(likes, 100); + assert_eq!(comments, 5); + assert_eq!(reposts, 2); + } + + #[test] + fn test_parse_tiktok_stats_k_suffix() { + let content = "**❤️\u{2000}203.1K\u{2000}💬\u{2000}718\u{2000}🔁\u{2000}4.8K**"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc, None); + assert_eq!(likes, 203_100); + assert_eq!(comments, 718); + assert_eq!(reposts, 4_800); + } + + #[test] + fn test_parse_tiktok_stats_bold_wrapped() { + let content = "Caption\n\n**❤️ 537 💬 3 🔁 13**"; + let (desc, likes, comments, reposts) = parse_tiktok_stats(content); + assert_eq!(desc.unwrap(), "Caption"); + assert_eq!(likes, 537); + assert_eq!(comments, 3); + assert_eq!(reposts, 13); + } +} From 4469839480143d7b04f162dd8c47c17f510f1919 Mon Sep 17 00:00:00 2001 From: CoreByte <62032154+CoreBytee@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:45:42 +0200 Subject: [PATCH 4/4] Limit workflow triggers to main branch only --- .github/workflows/build-binary.yml | 1 + .github/workflows/run-checks.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build-binary.yml b/.github/workflows/build-binary.yml index c7718f2..553a7a8 100644 --- a/.github/workflows/build-binary.yml +++ b/.github/workflows/build-binary.yml @@ -2,6 +2,7 @@ name: Build binary on: workflow_dispatch: push: + branches: [main] jobs: build-binary: name: Build binary diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml index d351148..53c3e07 100644 --- a/.github/workflows/run-checks.yml +++ b/.github/workflows/run-checks.yml @@ -2,6 +2,7 @@ name: Run checks on: workflow_dispatch: push: + branches: [main] pull_request: jobs: cargo-check: