Clean links - remove superfluous tracking elements from URLs

.env.example

@@ -50,3 +50,5 @@ REDLIB_DEFAULT_DISABLE_VISIT_REDDIT_CONFIRMATION=off
 REDLIB_DEFAULT_HIDE_SCORE=off
 # Enable fixed navbar by default
 REDLIB_DEFAULT_FIXED_NAVBAR=on
+# Enable tracking url removal
+REDLIB_DEFAULT_CLEAN_URLS=off

Cargo.toml

@@ -56,6 +56,7 @@ htmlescape = "0.3.1"
 bincode = "1.3.3"
 base2048 = "2.0.2"
 revision = "0.10.0"
+clearurls = "0.0.4"
 
 
 [dev-dependencies]

README.md

@@ -34,7 +34,7 @@
    - [Docker](#docker)
      - [Docker Compose](#docker-compose)
      - [Docker CLI](#docker-cli)
-   - Podman 
+   - Podman
       - Quadlets
 
    - [Binary](#binary)
@@ -216,7 +216,7 @@ Stream logs from the Redlib container:
 ```bash
 docker logs -f redlib
 ```
-## Podman 
+## Podman
 
 [Podman](https://podman.io/) lets you run containerized applications in a rootless fashion. Containers are loosely isolated environments that are lightweight and contain everything needed to run the application, so there's no need to rely on what's installed on the host.
 
@@ -225,8 +225,8 @@ Container images for Redlib are available at [quay.io](https://quay.io/repositor
 ### Quadlets
 
 > [!IMPORTANT]
-> These instructions assume that you are on a systemd based distro with [podman](https://podman.io/). If not, follow these [instructions on podman's website](https://podman.io/docs/installation) for how to do so. 
-> It also assumes you have used `loginctl enable-linger <username>` to enable the service to start for your user without logging in. 
+> These instructions assume that you are on a systemd based distro with [podman](https://podman.io/). If not, follow these [instructions on podman's website](https://podman.io/docs/installation) for how to do so.
+> It also assumes you have used `loginctl enable-linger <username>` to enable the service to start for your user without logging in.
 
 Copy the `redlib.container` and `.env.example` files to `.config/containers/systemd/` and modify any relevant values (for example, the ports Redlib should listen on, renaming the .env file and editing its values, etc.).
 
@@ -244,7 +244,7 @@ systemctl --user start redlib.service
 ```
 
 You can check the status of your container by using the following command:
-```bash 
+```bash
 systemctl --user status redlib.service
 ```
 
@@ -441,4 +441,5 @@ Assign a default value for each user-modifiable setting by passing environment v
 | `HIDE_SCORE`                        | `["on", "off"]`                                                                                                                    | `off`         |
 | `HIDE_SIDEBAR_AND_SUMMARY`          | `["on", "off"]`                                                                                                                    | `off`         |
 | `FIXED_NAVBAR`                      | `["on", "off"]`                                                                                                                    | `on`          |
-| `REMOVE_DEFAULT_FEEDS`              | `["on", "off"]`                                                                                                                    | `off`         |
+| `REMOVE_DEFAULT_FEEDS`              | `["on", "off"]`                                                                                                                    | `off`         |
+| `CLEAN_URLS`                        | `["on", "off"]`                                                                                                                    | `off`         |

app.json

@@ -79,6 +79,9 @@
     },
     "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS": {
       "required": false
+    },
+    "REDLIB_DEFAULT_CLEAN_URLS": {
+      "required": false
     }
   }
 }

src/config.rs

@@ -112,6 +112,9 @@ pub struct Config {
 
 	#[serde(rename = "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS")]
 	pub(crate) default_remove_default_feeds: Option<String>,
+
+	#[serde(rename = "REDLIB_DEFAULT_CLEAN_URLS")]
+	pub(crate) default_clean_urls: Option<String>,
 }
 
 impl Config {
@@ -160,6 +163,7 @@ impl Config {
 			enable_rss: parse("REDLIB_ENABLE_RSS"),
 			full_url: parse("REDLIB_FULL_URL"),
 			default_remove_default_feeds: parse("REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS"),
+			default_clean_urls: parse("REDLIB_DEFAULT_CLEAN_URLS"),
 		}
 	}
 }
@@ -190,6 +194,7 @@ fn get_setting_from_config(name: &str, config: &Config) -> Option<String> {
 		"REDLIB_ENABLE_RSS" => config.enable_rss.clone(),
 		"REDLIB_FULL_URL" => config.full_url.clone(),
 		"REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS" => config.default_remove_default_feeds.clone(),
+		"REDLIB_DEFAULT_CLEAN_URLS" => config.default_clean_urls.clone(),
 		_ => None,
 	}
 }

src/instance_info.rs

@@ -151,6 +151,7 @@ impl InstanceInfo {
 				["Hide HLS notification", &convert(&self.config.default_hide_hls_notification)],
 				["Subscriptions", &convert(&self.config.default_subscriptions)],
 				["Filters", &convert(&self.config.default_filters)],
+				["Clean URLs", &convert(&self.config.default_clean_urls)],
 			])
 			.with_header_row(["Default preferences"]),
 		);
@@ -186,6 +187,7 @@ impl InstanceInfo {
                     Default blur NSFW: {:?}\n
                     Default use HLS: {:?}\n
                     Default hide HLS notification: {:?}\n
+					Default clean urls: {:?}\n
                     Default subscriptions: {:?}\n
                     Default filters: {:?}\n",
 					self.package_name,
@@ -213,6 +215,7 @@ impl InstanceInfo {
 					self.config.default_blur_nsfw,
 					self.config.default_use_hls,
 					self.config.default_hide_hls_notification,
+					self.config.default_clean_urls,
 					self.config.default_subscriptions,
 					self.config.default_filters,
 				)

src/post.rs

@@ -6,7 +6,8 @@ use crate::config::get_setting;
 use crate::server::RequestExt;
 use crate::subreddit::{can_access_quarantine, quarantine};
 use crate::utils::{
-	error, format_num, get_filters, nsfw_landing, param, parse_post, rewrite_emotes, setting, template, time, val, Author, Awards, Comment, Flair, FlairPart, Post, Preferences,
+	clean_url, error, format_num, get_filters, nsfw_landing, param, parse_post, rewrite_emotes, setting, template, time, val, Author, Awards, Comment, Flair, FlairPart, Post,
+	Preferences,
 };
 use hyper::{Body, Request, Response};
 
@@ -64,7 +65,12 @@ pub async fn item(req: Request<Body>) -> Result<Response<Body>, String> {
 		// Otherwise, grab the JSON output from the request
 		Ok(response) => {
 			// Parse the JSON into Post and Comment structs
-			let post = parse_post(&response[0]["data"]["children"][0]).await;
+			let mut post = parse_post(&response[0]["data"]["children"][0]).await;
+
+			let clean_urls = setting(&req, "clean_urls");
+			if clean_urls == "on".to_owned() {
+				post.media.url = clean_url(post.media.url);
+			}
 
 			let req_url = req.uri().to_string();
 			// Return landing page if this post if this Reddit deems this post

src/search.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::cmp_owned)]
 
 // CRATES
-use crate::utils::{self, catch_random, error, filter_posts, format_num, format_url, get_filters, param, redirect, setting, template, val, Post, Preferences};
+use crate::utils::{self, catch_random, clean_url, error, filter_posts, format_num, format_url, get_filters, param, redirect, setting, template, val, Post, Preferences};
 use crate::{
 	client::json,
 	server::RequestExt,
@@ -131,6 +131,10 @@ pub async fn find(req: Request<Body>) -> Result<Response<Body>, String> {
 	} else {
 		match Post::fetch(&path, quarantined).await {
 			Ok((mut posts, after)) => {
+				let clean_urls = setting(&req, "clean_urls");
+				if clean_urls == "on".to_owned() {
+					posts.iter_mut().for_each(|post| post.media.url = clean_url(post.media.url.clone()));
+				}
 				let (_, all_posts_filtered) = filter_posts(&mut posts, &filters);
 				let no_posts = posts.is_empty();
 				let all_posts_hidden_nsfw = !no_posts && (posts.iter().all(|p| p.flags.nsfw) && setting(&req, "show_nsfw") != "on");

src/settings.rs

@@ -24,7 +24,7 @@ struct SettingsTemplate {
 
 // CONSTANTS
 
-const PREFS: [&str; 19] = [
+const PREFS: [&str; 20] = [
 	"theme",
 	"front_page",
 	"layout",
@@ -44,6 +44,7 @@ const PREFS: [&str; 19] = [
 	"disable_visit_reddit_confirmation",
 	"video_quality",
 	"remove_default_feeds",
+	"clean_urls",
 ];
 
 // FUNCTIONS
@@ -159,8 +160,8 @@ fn set_cookies_method(req: Request<Body>, remove_cookies: bool) -> Response<Body
 		.unwrap_or_else(String::new); // Return an empty string if None
 
 	// If there are subscriptions to restore set them and delete any old subscriptions cookies, otherwise delete them all
-	if subscriptions.is_some() {
-		let sub_list: Vec<String> = subscriptions.expect("Subscriptions").split('+').map(str::to_string).collect();
+	if let Some(subscriptions) = subscriptions {
+		let sub_list: Vec<String> = subscriptions.split('+').map(str::to_string).collect();
 
 		// Start at 0 to keep track of what number we need to start deleting old subscription cookies from
 		let mut subscriptions_number_to_delete_from = 0;
@@ -210,8 +211,8 @@ fn set_cookies_method(req: Request<Body>, remove_cookies: bool) -> Response<Body
 	}
 
 	// If there are filters to restore set them and delete any old filters cookies, otherwise delete them all
-	if filters.is_some() {
-		let filters_list: Vec<String> = filters.expect("Filters").split('+').map(str::to_string).collect();
+	if let Some(filters) = filters {
+		let filters_list: Vec<String> = filters.split('+').map(str::to_string).collect();
 
 		// Start at 0 to keep track of what number we need to start deleting old subscription cookies from
 		let mut filters_number_to_delete_from = 0;

src/subreddit.rs

@@ -3,8 +3,8 @@
 use crate::{config, utils};
 // CRATES
 use crate::utils::{
-	catch_random, error, filter_posts, format_num, format_url, get_filters, info, nsfw_landing, param, redirect, rewrite_urls, setting, template, val, Post, Preferences,
-	Subreddit,
+	catch_random, clean_url, error, filter_posts, format_num, format_url, get_filters, info, nsfw_landing, param, redirect, rewrite_urls, setting, template, val, Post,
+	Preferences, Subreddit,
 };
 use crate::{client::json, server::RequestExt, server::ResponseExt};
 use cookie::Cookie;
@@ -164,6 +164,10 @@ pub async fn community(req: Request<Body>) -> Result<Response<Body>, String> {
 	} else {
 		match Post::fetch(&path, quarantined).await {
 			Ok((mut posts, after)) => {
+				let clean_urls = setting(&req, "clean_urls");
+				if clean_urls == "on".to_owned() {
+					posts.iter_mut().for_each(|post| post.media.url = clean_url(post.media.url.clone()));
+				}
 				let (_, all_posts_filtered) = filter_posts(&mut posts, &filters);
 				let no_posts = posts.is_empty();
 				let all_posts_hidden_nsfw = !no_posts && (posts.iter().all(|p| p.flags.nsfw) && setting(&req, "show_nsfw") != "on");

src/utils.rs

@@ -6,6 +6,7 @@ use crate::config::{self, get_setting};
 // CRATES
 //
 use crate::{client::json, server::RequestExt};
+use clearurls::UrlCleaner;
 use cookie::Cookie;
 use hyper::{Body, Request, Response};
 use libflate::deflate::{Decoder, Encoder};
@@ -23,6 +24,7 @@ use std::env;
 use std::io::{Read, Write};
 use std::str::FromStr;
 use std::string::ToString;
+use std::sync::Mutex;
 use time::{macros::format_description, Duration, OffsetDateTime};
 use url::Url;
 
@@ -670,6 +672,8 @@ pub struct Preferences {
 	pub hide_score: String,
 	#[revision(start = 1)]
 	pub remove_default_feeds: String,
+	#[revision(start = 1)]
+	pub clean_urls: String,
 }
 
 fn serialize_vec_with_plus<S>(vec: &[String], serializer: S) -> Result<S::Ok, S::Error>
@@ -728,6 +732,7 @@ impl Preferences {
 			hide_awards: setting(req, "hide_awards"),
 			hide_score: setting(req, "hide_score"),
 			remove_default_feeds: setting(req, "remove_default_feeds"),
+			clean_urls: setting(req, "clean_urls"),
 		}
 	}
 
@@ -1075,6 +1080,22 @@ pub fn format_url(url: &str) -> String {
 	}
 }
 
+// Remove tracking query params
+static URL_CLEANER: Lazy<Mutex<UrlCleaner>> = Lazy::new(|| Mutex::new(UrlCleaner::from_embedded_rules().expect("Failed to initialize UrlCleaner")));
+
+pub fn clean_url(url: String) -> String {
+	let is_external_url = match Url::parse(url.as_str()) {
+		Ok(parsed_url) => parsed_url.domain().is_some(),
+		_ => false,
+	};
+	let mut cleaned_url = url.clone();
+	if is_external_url {
+		let cleaner = URL_CLEANER.lock().unwrap();
+		cleaned_url = cleaner.clear_single_url_str(cleaned_url.as_str()).expect("Unable to clean the URL.").as_ref().to_owned();
+	}
+	cleaned_url
+}
+
 static REGEX_BULLET: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?m)^- (.*)$").unwrap());
 static REGEX_BULLET_CONSECUTIVE_LINES: Lazy<Regex> = Lazy::new(|| Regex::new(r"</ul>\n<ul>").unwrap());
 
@@ -1423,13 +1444,11 @@ pub async fn nsfw_landing(req: Request<Body>, req_url: String) -> Result<Respons
 pub fn url_path_basename(path: &str) -> String {
 	let url_result = Url::parse(format!("https://libredd.it/{path}").as_str());
 
-	if url_result.is_err() {
-		path.to_string()
-	} else {
-		let mut url = url_result.unwrap();
+	if let Ok(mut url) = url_result {
 		url.path_segments_mut().unwrap().pop_if_empty();
-
 		url.path_segments().unwrap().next_back().unwrap().to_string()
+	} else {
+		path.to_string()
 	}
 }
 
@@ -1541,10 +1560,11 @@ mod tests {
 			hide_awards: "off".to_owned(),
 			hide_score: "off".to_owned(),
 			remove_default_feeds: "off".to_owned(),
+			clean_urls: "off".to_owned(),
 		};
 		let urlencoded = serde_urlencoded::to_string(prefs).expect("Failed to serialize Prefs");
 
-		assert_eq!(urlencoded, "theme=laserwave&front_page=default&layout=compact&wide=on&blur_spoiler=on&show_nsfw=off&blur_nsfw=on&hide_hls_notification=off&video_quality=best&hide_sidebar_and_summary=off&use_hls=on&autoplay_videos=on&fixed_navbar=on&disable_visit_reddit_confirmation=on&comment_sort=confidence&post_sort=top&subscriptions=memes%2Bmildlyinteresting&filters=&hide_awards=off&hide_score=off&remove_default_feeds=off");
+		assert_eq!(urlencoded, "theme=laserwave&front_page=default&layout=compact&wide=on&blur_spoiler=on&show_nsfw=off&blur_nsfw=on&hide_hls_notification=off&video_quality=best&hide_sidebar_and_summary=off&use_hls=on&autoplay_videos=on&fixed_navbar=on&disable_visit_reddit_confirmation=on&comment_sort=confidence&post_sort=top&subscriptions=memes%2Bmildlyinteresting&filters=&hide_awards=off&hide_score=off&remove_default_feeds=off&clean_urls=off");
 	}
 }
 
@@ -1631,7 +1651,7 @@ fn test_rewriting_bullet_list() {
 How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge&#39;s icon for example, the blue background is just blocky, don&#39;t know why.</p>
 </div>"#;
 	let output = r#"<div class="md"><p>Hi, I&#39;ve bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I&#39;ve installed its driver from the LG website and it works ok. I also used <a href="http://www.lagom.nl/lcd-test/">http://www.lagom.nl/lcd-test/</a> to calibrate it. After some good tinkering I&#39;ve found the following settings + the color profile from the driver gets me past all the tests perfectly:
-<ul><li>Brightness 50 (still have to settle on this one, it&#39;s personal preference, it controls the backlight, not the colors)</li><li>Contrast 70 (which for me was the default one)</li><li>Picture mode Custom</li><li>Super resolution + Off (it looks horrible anyway)</li><li>Sharpness 50 (default one I think)</li><li>Black level High (low messes up gray colors)</li><li>DFC Off </li><li>Response Time Middle (personal preference, <a href="https://www.blurbusters.com/">https://www.blurbusters.com/</a> show horrible overdrive with it on high)</li><li>Freesync doesn&#39;t matter</li><li>Black stabilizer 50</li><li>Gamma setting on 0 </li><li>Color Temp Medium</li></ul>
+<ul><li>Brightness 50 (still have to settle on this one, it&#39;s personal preference, it controls the backlight, not the colors)</li><li>Contrast 70 (which for me was the default one)</li><li>Picture mode Custom</li><li>Super resolution + Off (it looks horrible anyway)</li><li>Sharpness 50 (default one I think)</li><li>Black level High (low messes up gray colors)</li><li>DFC Off</li><li>Response Time Middle (personal preference, <a href="https://www.blurbusters.com/">https://www.blurbusters.com/</a> show horrible overdrive with it on high)</li><li>Freesync doesn&#39;t matter</li><li>Black stabilizer 50</li><li>Gamma setting on 0</li><li>Color Temp Medium</li></ul>
 How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge&#39;s icon for example, the blue background is just blocky, don&#39;t know why.</p>
 </div>"#;
 
@@ -1654,9 +1674,9 @@ fn test_default_prefs_serialization_loop_bincode() {
 }
 
 static KNOWN_GOOD_CONFIGS: &[&str] = &[
-	"ఴӅβØØҞÉဏႢձĬ༧ȒʯऌԔӵ୮༏",
-	"ਧՊΥÀÃǎƱГ۸ඣമĖฤ႙ʟาúໜϾௐɥঀĜໃહཞઠѫҲɂఙ࿔DzઉƲӟӻĻฅΜδ໖ԜǗဖငƦơ৶Ą௩ԹʛใЛʃශаΏ",
-	"ਧԩΥÀÃΊ౭൩ඔႠϼҭöҪƸռઇԾॐნɔາǒՍҰच௨ಖມŃЉŐདƦ๙ϩএఠȝഽйʮჯඒϰळՋ௮ສ৵ऎΦѧਹಧଟƙŃ३î༦ŌပղयƟแҜ།",
+	"ਧӐΥºÃΦĴгౡୡϤҚԷŽဎՐΧΣೡຽဒ೨ʛĽତ๘Ӓǹভµɾ൦ॴцৱ௬చΣҭжҭȱȾཊజĊȔ௸७ƘȂј۰ȥėǨԯၻíႽਈႴ۹ଆ",
+	"ਧҫടºÃǒɣυໃਣөŕǁజ८ௐɪDžઘႴ౨ඛႻຫǪၼդɍ৪Êѕ୶ʭѹŪҚຊೱѰງიŠСঌາඌĨğਜડ࿅ଠಲೱҋŇƞਭăʁझшȖǾཔ௧ந۞ສÚ",
+	"ਧҫടºÃǒɿဧϯljഔค๖۞ԆНȦ൨ĭ྅ҤƍตཧႯƅशञঊମਇȕමзқଽijჰଐՋບӎՓஶཕ૭ଛกήऋĜɀಱӔԩझԩîဓŒԬũլಙટщೞຝ৪༎",
 ];
 
 #[test]

templates/settings.html

@@ -145,6 +145,12 @@
 					<input type="checkbox" name="disable_visit_reddit_confirmation" {% if
 						prefs.disable_visit_reddit_confirmation=="on" %}checked{% endif %}>
 				</div>
+				<div class="prefs-group">
+					<label for="clean_urls">Remove tracking elements from URLs</label>
+					<input type="hidden" value="off" name="clean_urls">
+					<input type="checkbox" name="clean_urls" {% if
+						prefs.clean_urls=="on" %}checked{% endif %}>
+				</div>
 			</fieldset>
 			<input id="save" type="submit" value="Save">
 		</div>
@@ -214,4 +220,4 @@
 	</div>
 </div>
 
-{% endblock %}
+{% endblock %}