diff --git a/Cargo.lock b/Cargo.lock index 9ac47ebe4..75b85ae8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,7 @@ dependencies = [ "primitives", "rand 0.8.5", "reqwest", + "scraper", "serde", "serde_json", "thiserror", @@ -335,7 +336,7 @@ dependencies = [ "http", "http-body", "hyper", - "itoa", + "itoa 1.0.4", "matchit", "memchr", "mime", @@ -610,7 +611,7 @@ checksum = "29c39203181991a7dd4343b8005bd804e7a9a37afb8ac070e43771e8c820bbde" dependencies = [ "chrono", "chrono-tz-build", - "phf", + "phf 0.11.1", ] [[package]] @@ -620,8 +621,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f509c3a87b33437b05e2458750a0700e5bdd6956176773e6c7d6dd15a283a0c" dependencies = [ "parse-zoneinfo", - "phf", - "phf_codegen", + "phf 0.11.1", + "phf_codegen 0.11.1", ] [[package]] @@ -846,6 +847,33 @@ dependencies = [ "subtle", ] +[[package]] +name = "cssparser" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa 0.4.8", + "matches", + "phf 0.8.0", + "proc-macro2", + "quote", + "smallvec 1.10.0", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "ctor" version = "0.1.23" @@ -1083,6 +1111,27 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "dtoa-short" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + [[package]] name = "either" version = "1.8.0" @@ -1305,6 +1354,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.24" @@ -1415,6 +1474,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder 1.4.3", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -1425,6 +1493,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1573,6 +1650,20 @@ dependencies = [ "digest 0.10.5", ] +[[package]] +name = "html5ever" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "http" version = "0.2.8" @@ -1581,7 +1672,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.4", ] [[package]] @@ -1661,7 +1752,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.4", "pin-project-lite", "socket2", "tokio", @@ -1831,6 +1922,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.4" @@ -1927,6 +2024,26 @@ dependencies = [ "value-bag", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +dependencies = [ + "log", + "phf 0.10.1", + "phf_codegen 0.10.0", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "matches" version = "0.1.9" @@ -2102,6 +2219,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + [[package]] name = "nodrop" version = "0.1.14" @@ -2499,13 +2622,53 @@ dependencies = [ "sha1 0.10.5", ] +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_macros", + "phf_shared 0.8.0", + "proc-macro-hack", +] + +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", +] + [[package]] name = "phf" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" dependencies = [ - "phf_shared", + "phf_shared 0.11.1", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", +] + +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", ] [[package]] @@ -2514,8 +2677,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.1", + "phf_shared 0.11.1", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared 0.8.0", + "rand 0.7.3", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand 0.8.5", ] [[package]] @@ -2524,10 +2707,42 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" dependencies = [ - "phf_shared", + "phf_shared 0.11.1", "rand 0.8.5", ] +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.11.1" @@ -2680,6 +2895,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "pretty_assertions" version = "1.3.0" @@ -2853,6 +3074,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", + "rand_pcg", ] [[package]] @@ -2913,6 +3135,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rayon" version = "1.5.3" @@ -2947,7 +3178,7 @@ dependencies = [ "bytes", "combine", "futures-util", - "itoa", + "itoa 1.0.4", "percent-encoding", "pin-project-lite", "ryu", @@ -3173,6 +3404,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scraper" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5684396b456f3eb69ceeb34d1b5cb1a2f6acf7ca4452131efa3ba0ee2c2d0a70" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "matches", + "selectors", + "smallvec 1.10.0", + "tendril", +] + [[package]] name = "scratch" version = "1.0.2" @@ -3255,6 +3502,26 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "matches", + "phf 0.8.0", + "phf_codegen 0.8.0", + "precomputed-hash", + "servo_arc", + "smallvec 1.10.0", + "thin-slice", +] + [[package]] name = "semver" version = "1.0.14" @@ -3335,7 +3602,7 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" dependencies = [ - "itoa", + "itoa 1.0.4", "ryu", "serde", ] @@ -3378,7 +3645,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa", + "itoa 1.0.4", "ryu", "serde", ] @@ -3411,6 +3678,16 @@ dependencies = [ "syn", ] +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + [[package]] name = "sha-1" version = "0.9.8" @@ -3606,12 +3883,44 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "string_cache" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + [[package]] name = "stringprep" version = "0.1.2" @@ -3722,6 +4031,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "tera" version = "1.17.1" @@ -3791,6 +4111,12 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + [[package]] name = "thiserror" version = "1.0.37" @@ -3837,7 +4163,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c" dependencies = [ - "itoa", + "itoa 1.0.4", "libc", "num_threads", "serde", @@ -3930,7 +4256,7 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf", + "phf 0.11.1", "pin-project-lite", "postgres-protocol", "postgres-types", @@ -4219,6 +4545,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "uuid" version = "1.2.1" diff --git a/adview-manager/Cargo.toml b/adview-manager/Cargo.toml index feab96eda..c58671b64 100644 --- a/adview-manager/Cargo.toml +++ b/adview-manager/Cargo.toml @@ -33,3 +33,4 @@ rand = "0.8" adex_primitives = { version = "0.2", path = "../primitives", package = "primitives", features = ["test-util"] } wiremock = "0.5" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +scraper = "0.13" diff --git a/adview-manager/src/helpers.rs b/adview-manager/src/helpers.rs index b6995de29..5e0103d4b 100644 --- a/adview-manager/src/helpers.rs +++ b/adview-manager/src/helpers.rs @@ -163,7 +163,7 @@ pub fn get_unit_html_with_events( .iter() .map(|validator| { let fetch_url = format!( - "{}/campaign/{}/events?pubAddr={}", + "{}/v5/campaign/{}/events?pubAddr={}", validator.url, campaign_id, options.publisher_addr ); @@ -201,7 +201,13 @@ pub fn get_unit_html_with_events( #[cfg(test)] mod test { use super::*; - use adex_primitives::test_util::DUMMY_IPFS; + use adex_primitives::{ + config::GANACHE_CONFIG, + test_util::{DUMMY_CAMPAIGN, DUMMY_IPFS, PUBLISHER}, + util::ApiUrl, + }; + use scraper::{Html, Selector}; + use std::collections::HashSet; fn get_ad_unit(media_mime: &str) -> AdUnit { AdUnit { @@ -231,6 +237,188 @@ mod test { assert_eq!("http://123".to_string(), normalize_url("http://123")); } + #[test] + fn getting_unit_html() { + let size_1 = Size { + width: 480, + height: 480, + }; + + let size_2 = Size { + width: 920, + height: 160, + }; + + let video_unit = AdUnit { + ipfs: DUMMY_IPFS[0], + media_url: "".into(), + media_mime: "video/avi".into(), + target_url: "https://ambire.com?utm_source=adex_PUBHOSTNAME".into(), + }; + + let image_unit = AdUnit { + ipfs: DUMMY_IPFS[1], + media_url: "".into(), + media_mime: "image/jpeg".into(), + target_url: "https://ambire.com?utm_source=adex_PUBHOSTNAME".into(), + }; + + // Test for first size, correct link, video inside link + { + let unit = get_unit_html(Some(size_1), &video_unit, "https://adex.network", "", ""); + let fragment = Html::parse_fragment(&unit); + + let div_selector = Selector::parse("div").unwrap(); + let div = fragment + .select(&div_selector) + .next() + .expect("There should be a div"); + + let anchor_selector = Selector::parse("div>a").unwrap(); + let anchor = fragment + .select(&anchor_selector) + .next() + .expect("There should be an anchor"); + + let video_selector = Selector::parse("div>a>video").unwrap(); + let video = fragment + .select(&video_selector) + .next() + .expect("There should be a video"); + + assert_eq!("div", div.value().name()); + + assert_eq!("a", anchor.value().name()); + assert_eq!( + Some("https://ambire.com?utm_source=AdEx+(https://adex.network)"), + anchor.value().attr("href") + ); + + assert_eq!("video", video.value().name()); + assert_eq!(Some("480"), video.value().attr("width")); + assert_eq!(Some("480"), video.value().attr("height")); + } + // Test for another size + { + let unit = get_unit_html(Some(size_2), &video_unit, "https://adex.network", "", ""); + let fragment = Html::parse_fragment(&unit); + + let video_selector = Selector::parse("div>a>video").unwrap(); + let video = fragment + .select(&video_selector) + .next() + .expect("There should be a video"); + + assert_eq!("video", video.value().name()); + assert_eq!(Some("920"), video.value().attr("width")); + assert_eq!(Some("160"), video.value().attr("height")); + } + + // Test for image ad_unit + { + let unit = get_unit_html(Some(size_1), &image_unit, "https://adex.network", "", ""); + let fragment = Html::parse_fragment(&unit); + + let image_selector = Selector::parse("div>a>*").unwrap(); + let image = fragment + .select(&image_selector) + .next() + .expect("There should be an image"); + + assert_eq!("img", image.value().name()); + } + + // Test for another hostname + { + let unit = get_unit_html( + Some(size_1), + &video_unit, + "https://adsense.google.com", + "", + "", + ); + let fragment = Html::parse_fragment(&unit); + + let anchor_selector = Selector::parse("div>a").unwrap(); + let anchor = fragment + .select(&anchor_selector) + .next() + .expect("There should be a second anchor"); + + assert_eq!("a", anchor.value().name()); + assert_eq!( + Some("https://ambire.com?utm_source=AdEx+(https://adsense.google.com)"), + anchor.value().attr("href") + ); + } + } + + #[test] + fn getting_unit_html_with_events() { + let whitelisted_tokens = GANACHE_CONFIG + .chains + .values() + .flat_map(|chain| chain.tokens.values().map(|token| token.address)) + .collect::>(); + + let market_url = ApiUrl::parse("https://market.adex.network").expect("should parse"); + let validator_1_url = ApiUrl::parse("https://tom.adex.network").expect("should parse"); + let validator_2_url = ApiUrl::parse("https://jerry.adex.network").expect("should parse"); + let options = Options { + market_url, + market_slot: DUMMY_IPFS[0], + publisher_addr: *PUBLISHER, + // All passed tokens must be of the same price and decimals, so that the amounts can be accurately compared + whitelisted_tokens, + size: Some(Size::new(300, 100)), + navigator_language: Some("bg".into()), + disabled_video: false, + disabled_sticky: false, + validators: vec![validator_1_url, validator_2_url], + }; + let ad_unit = AdUnit { + ipfs: DUMMY_IPFS[0], + media_url: "".into(), + media_mime: "video/avi".into(), + target_url: "https://ambire.com?utm_source=adex_PUBHOSTNAME".into(), + }; + let campaign_id = DUMMY_CAMPAIGN.id; + let validators = DUMMY_CAMPAIGN.validators.clone(); + // Test with events + { + let unit_with_events = get_unit_html_with_events( + &options, + &ad_unit, + "https://adex.network", + campaign_id, + &validators, + false, + ); + let fragment = Html::parse_fragment(&unit_with_events); + + let anchor_selector = Selector::parse("div>a").unwrap(); + let anchor = fragment + .select(&anchor_selector) + .next() + .expect("There should be a second anchor"); + + let video_selector = Selector::parse("div>a>video").unwrap(); + let video = fragment + .select(&video_selector) + .next() + .expect("There should be a video"); + + let expected_onclick: &str = &format!("var fetchOpts = {{ method: 'POST', headers: {{ 'content-type': 'application/json' }}, body: {{'events':[{{'type':'CLICK','publisher':'{}','adUnit':'{}','adSlot':'{}','referrer':'document.referrer'}}]}} }}; fetch('{}/v5/campaign/{}/events?pubAddr={}', fetchOpts); fetch('{}/v5/campaign/{}/events?pubAddr={}', fetchOpts)", options.publisher_addr, ad_unit.ipfs, options.market_slot, validators[0].url, campaign_id, options.publisher_addr, validators[1].url, campaign_id, options.publisher_addr); + assert_eq!(Some(expected_onclick), anchor.value().attr("onclick")); + + let expected_onloadeddata: &str = &format!("setTimeout(function() {{ var fetchOpts = {{ method: 'POST', headers: {{ 'content-type': 'application/json' }}, body: {{'events':[{{'type':'IMPRESSION','publisher':'{}','adUnit':'{}','adSlot':'{}','referrer':'document.referrer'}}]}} }}; fetch('{}/v5/campaign/{}/events?pubAddr={}', fetchOpts); fetch('{}/v5/campaign/{}/events?pubAddr={}', fetchOpts) }}, 8000)", options.publisher_addr, ad_unit.ipfs, options.market_slot, validators[0].url, campaign_id, options.publisher_addr, validators[1].url, campaign_id, options.publisher_addr); + assert_eq!( + Some(expected_onloadeddata), + video.value().attr("onloadeddata") + ); + } + } + mod randomized_sort_pos { use super::*; diff --git a/adview-manager/src/manager.rs b/adview-manager/src/manager.rs index 05ad490a2..188ffa452 100644 --- a/adview-manager/src/manager.rs +++ b/adview-manager/src/manager.rs @@ -151,13 +151,8 @@ impl Manager { mut input: input::Input, campaign_id: CampaignId, ) -> input::Input { - let seconds_since_campaign_impression = self - .history - .read() - .await - .iter() - .rev() - .find_map(|h| { + let seconds_since_campaign_impression = + self.history.read().await.iter().rev().find_map(|h| { if h.campaign_id == campaign_id { let last_impression: Duration = Utc::now() - h.time; @@ -165,8 +160,7 @@ impl Manager { } else { None } - }) - .unwrap_or(u64::MAX); + }); input.ad_view = Some(input::AdView { seconds_since_campaign_impression, @@ -203,17 +197,13 @@ impl Manager { .iter() .find(|c| c.campaign.id == sticky_entry.campaign_id)?; - let unit = stick_campaign - .units_with_price - .iter() - .find_map(|u| { - if u.unit.ipfs == sticky_entry.unit_id { - Some(u.unit.clone()) - } else { - None - } - }) - .expect("Something went terribly wrong. Data is corrupted! There should be an AdUnit"); + let unit = stick_campaign.units_with_price.iter().find_map(|u| { + if u.unit.ipfs == sticky_entry.unit_id { + Some(u.unit.clone()) + } else { + None + } + })?; let html = get_unit_html_with_events( &self.options, @@ -293,7 +283,7 @@ impl Manager { pub async fn get_next_ad_unit(&self) -> Result, Error> { let units_for_slot = self.get_units_for_slot_resp().await?; - let m_campaigns = &units_for_slot.campaigns; + let ufs_campaigns = &units_for_slot.campaigns; let fallback_unit = units_for_slot.fallback_unit; let targeting_input = units_for_slot.targeting_input_base; @@ -305,7 +295,7 @@ impl Manager { // Stickiness is when we keep showing an ad unit for a slot for some time in order to achieve fair impression value // see https://github.com/AdExNetwork/adex-adview-manager/issues/65 - let sticky_result = self.get_sticky_ad_unit(m_campaigns, &hostname).await; + let sticky_result = self.get_sticky_ad_unit(ufs_campaigns, &hostname).await; if let Some(sticky) = sticky_result { return Ok(Some(NextAdUnit { unit: sticky.unit, @@ -322,19 +312,19 @@ impl Manager { let seed = BigNum::from(random as u64); // Apply targeting, now with adView.* variables, and sort the resulting ad units - let mut units_with_price = m_campaigns + let mut units_with_price = ufs_campaigns .iter() - .flat_map(|m_campaign| { + .flat_map(|ufs_campaign| { // since we are in a Iterator.map(), we can't use async, so we block - if block_on(self.is_campaign_sticky(m_campaign.campaign.id)) { + if block_on(self.is_campaign_sticky(ufs_campaign.campaign.id)) { return vec![]; } - let campaign_id = m_campaign.campaign.id; + let campaign_id = ufs_campaign.campaign.id; - let mut unit_input = targeting_input.clone().with_campaign(m_campaign.campaign.clone()); + let mut unit_input = targeting_input.clone().with_campaign(ufs_campaign.campaign.clone()); - m_campaign + ufs_campaign .units_with_price .iter() .filter(|unit_with_price| { @@ -349,7 +339,7 @@ impl Manager { let on_type_error = |type_error, rule| error!(target: "rule-evaluation", "Rule evaluation error for {campaign_id:?}, {rule:?} with error: {type_error:?}"); targeting::eval_with_callback( - &m_campaign.campaign.targeting_rules, + &ufs_campaign.campaign.targeting_rules, &unit_input, &mut output, Some(on_type_error) @@ -397,11 +387,11 @@ impl Manager { // Return the results, with a fallback unit if there is one if let Some((unit_with_price, campaign_id)) = auction_winner { - let validators = m_campaigns + let validators = ufs_campaigns .iter() - .find_map(|m_campaign| { - if &m_campaign.campaign.id == campaign_id { - Some(&m_campaign.campaign.validators) + .find_map(|ufs_campaign| { + if &ufs_campaign.campaign.id == campaign_id { + Some(&ufs_campaign.campaign.validators) } else { None } @@ -443,14 +433,46 @@ mod test { use super::*; use crate::manager::input::Input; use adex_primitives::{ - sentry::CLICK, - test_util::{CAMPAIGNS, DUMMY_AD_UNITS, DUMMY_IPFS, PUBLISHER}, + config::GANACHE_CONFIG, + sentry::{ + units_for_slot::response::{AdUnit, UnitsWithPrice}, + CLICK, + }, + test_util::{CAMPAIGNS, DUMMY_AD_UNITS, DUMMY_CAMPAIGN, DUMMY_IPFS, PUBLISHER}, + unified_num::FromWhole, }; use wiremock::{ matchers::{method, path}, Mock, MockServer, ResponseTemplate, }; + fn setup_manager(uri: String) -> Manager { + let market_url = uri.parse().unwrap(); + let whitelisted_tokens = GANACHE_CONFIG + .chains + .values() + .flat_map(|chain| chain.tokens.values().map(|token| token.address)) + .collect::>(); + + let validator_1_url = ApiUrl::parse(&format!("{}/validator-1", uri)).expect("should parse"); + let validator_2_url = ApiUrl::parse(&format!("{}/validator-2", uri)).expect("should parse"); + let validator_3_url = ApiUrl::parse(&format!("{}/validator-3", uri)).expect("should parse"); + + let options = Options { + market_url, + market_slot: DUMMY_IPFS[0], + publisher_addr: *PUBLISHER, + // All passed tokens must be of the same price and decimals, so that the amounts can be accurately compared + whitelisted_tokens, + size: Some(Size::new(300, 100)), + navigator_language: Some("bg".into()), + disabled_video: false, + disabled_sticky: false, + validators: vec![validator_1_url, validator_2_url, validator_3_url], + }; + + Manager::new(options.clone(), Default::default()).expect("Failed to create AdView Manager") + } #[tokio::test] async fn test_querying_for_units_for_slot() { // 1. Set up mock servers for each validator @@ -573,30 +595,7 @@ mod test { .await; // 2. Set up a manager - let market_url = server.uri().parse().unwrap(); - let whitelisted_tokens = DEFAULT_TOKENS.clone(); - - let validator_1_url = - ApiUrl::parse(&format!("{}/validator-1", server.uri())).expect("should parse"); - let validator_2_url = - ApiUrl::parse(&format!("{}/validator-2", server.uri())).expect("should parse"); - let validator_3_url = - ApiUrl::parse(&format!("{}/validator-3", server.uri())).expect("should parse"); - let options = Options { - market_url, - market_slot: DUMMY_IPFS[0], - publisher_addr: *PUBLISHER, - // All passed tokens must be of the same price and decimals, so that the amounts can be accurately compared - whitelisted_tokens, - size: Some(Size::new(300, 100)), - navigator_language: Some("bg".into()), - disabled_video: false, - disabled_sticky: false, - validators: vec![validator_1_url, validator_2_url, validator_3_url], - }; - - let manager = Manager::new(options.clone(), Default::default()) - .expect("Failed to create AdView Manager"); + let manager = setup_manager(server.uri()); let res = manager .get_units_for_slot_resp() @@ -607,4 +606,86 @@ mod test { assert_eq!(res.fallback_unit, Some(original_ad_unit)); assert_eq!(res.campaigns, vec![campaign_0, campaign_1, campaign_2]); } + + #[tokio::test] + async fn check_if_campaign_is_sticky() { + let mut manager = setup_manager("http://localhost:1337".to_string()); + + // Case 1 - options has disabled sticky + { + manager.options.disabled_sticky = true; + assert!(!manager.is_campaign_sticky(DUMMY_CAMPAIGN.id).await); + manager.options.disabled_sticky = false; + } + // Case 2 - time is past stickiness treshold, less than 4 minutes ago + { + let history = vec![HistoryEntry { + time: Utc::now() - Duration::days(1), // 24 hours ago + unit_id: DUMMY_IPFS[0], + campaign_id: DUMMY_CAMPAIGN.id, + slot_id: DUMMY_IPFS[1], + }]; + let history = Arc::new(RwLock::new(VecDeque::from(history))); + + manager.history = history; + + assert!(!manager.is_campaign_sticky(DUMMY_CAMPAIGN.id).await); + } + // Case 3 - time isn't past stickiness treshold, Utc::now() + { + let history = vec![HistoryEntry { + time: Utc::now(), + unit_id: DUMMY_IPFS[0], + campaign_id: DUMMY_CAMPAIGN.id, + slot_id: DUMMY_IPFS[1], + }]; + let history = Arc::new(RwLock::new(VecDeque::from(history))); + + manager.history = history; + + assert!(manager.is_campaign_sticky(DUMMY_CAMPAIGN.id).await); + } + } + + #[tokio::test] + async fn check_sticky_ad_unit() { + let server = MockServer::start().await; + let mut manager = setup_manager(server.uri()); + let history = vec![HistoryEntry { + time: Utc::now(), + unit_id: DUMMY_AD_UNITS[0].ipfs, + campaign_id: DUMMY_CAMPAIGN.id, + slot_id: manager.options.market_slot, + }]; + let history = Arc::new(RwLock::new(VecDeque::from(history))); + manager.history = history; + + let campaign = Campaign { + campaign: DUMMY_CAMPAIGN.clone(), + units_with_price: vec![UnitsWithPrice { + unit: AdUnit::from(&DUMMY_AD_UNITS[0]), + price: UnifiedNum::from_whole(0.0001), + }], + }; + let res = manager + .get_sticky_ad_unit(&[campaign], "http://localhost:1337") + .await; + + assert!(res.is_some()); + + // TODO: Here we modify the campaign manually, verify that such a scenario is possible + let campaign = Campaign { + campaign: DUMMY_CAMPAIGN.clone(), + units_with_price: vec![UnitsWithPrice { + unit: AdUnit::from(&DUMMY_AD_UNITS[1]), + price: UnifiedNum::from_whole(0.0001), + }], + }; + + let res = manager + .get_sticky_ad_unit(&[campaign], "http://localhost:1337") + .await; + + assert!(res.is_none()); + } } diff --git a/primitives/src/targeting/eval_test.rs b/primitives/src/targeting/eval_test.rs index 655c8fdea..da2c943ec 100644 --- a/primitives/src/targeting/eval_test.rs +++ b/primitives/src/targeting/eval_test.rs @@ -13,7 +13,7 @@ fn get_default_input() -> Input { let init_input = Input { ad_view: Some(input::AdView { - seconds_since_campaign_impression: 10, + seconds_since_campaign_impression: Some(10), has_custom_preferences: false, navigator_language: "bg".to_string(), }), diff --git a/primitives/src/targeting/input.rs b/primitives/src/targeting/input.rs index 5ba911915..abd2e9ea2 100644 --- a/primitives/src/targeting/input.rs +++ b/primitives/src/targeting/input.rs @@ -116,7 +116,7 @@ impl GetField for Input { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct AdView { - pub seconds_since_campaign_impression: u64, + pub seconds_since_campaign_impression: Option, pub has_custom_preferences: bool, pub navigator_language: String, } @@ -127,9 +127,13 @@ impl GetField for AdView { fn get(&self, field: &Self::Field) -> Self::Output { match field { - field::AdView::SecondsSinceCampaignImpression => { - Value::Number(self.seconds_since_campaign_impression.into()) - } + // We could use `Option`, however, `try_get` will return `UnknownVariable` + // this is why we use `u64::MAX` when returning the value of the field. + field::AdView::SecondsSinceCampaignImpression => Value::Number( + self.seconds_since_campaign_impression + .unwrap_or(u64::MAX) + .into(), + ), field::AdView::HasCustomPreferences => Value::Bool(self.has_custom_preferences), field::AdView::NavigatorLanguage => Value::String(self.navigator_language.clone()), } @@ -418,7 +422,7 @@ mod test { let full_input = Input { ad_view: Some(AdView { - seconds_since_campaign_impression: 10, + seconds_since_campaign_impression: Some(10), has_custom_preferences: true, navigator_language: "en".into(), }),