Mock HTTP requests in tests (#898)

Certain tests involving HTTP requests were sometimes hanging
indefinitely, so this uses Mockito for HTTP mocking. This seemingly
resolves the issue and makes these tests more reliable.

The existing can_fail_404_links test has been renamed to
can_fail_unresolved_links, to represent what actually occurs in the
test. The can_fail_404_links test now deals with a proper 404
response.

Just to be clear, the check_site test in the site component will
still create outgoing HTTP requests (due to the URLs used in the
test_site), so this commit only uses HTTP mocking where possible.

Cargo.lock

@@ -295,6 +295,15 @@ name = "arc-swap"
 version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "assert-json-diff"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.13"
@@ -471,6 +480,16 @@ name = "color_quant"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "colored"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "config"
 version = "0.1.0"
@@ -678,6 +697,11 @@ name = "deunicode"
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "difference"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "digest"
 version = "0.8.1"
@@ -1295,6 +1319,7 @@ dependencies = [
  "config 0.1.0",
  "errors 0.1.0",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mockito 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1456,6 +1481,23 @@ dependencies = [
  "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "mockito"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "assert-json-diff 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "colored 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "native-tls"
 version = "0.2.3"
@@ -2558,6 +2600,7 @@ dependencies = [
  "imageproc 0.1.0",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "library 0.1.0",
+ "mockito 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pulldown-cmark 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3251,6 +3294,7 @@ dependencies = [
 "checksum ammonia 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e266e1f4be5ffa05309f650e2586fe1d3ae6034eb24025a7ae1dfecc330823a"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
 "checksum arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff"
+"checksum assert-json-diff 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9881d306dee755eccf052d652b774a6b2861e86b4772f555262130e58e4f81d2"
 "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
 "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
 "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
@@ -3273,6 +3317,7 @@ dependencies = [
 "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
 "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
 "checksum color_quant 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd"
+"checksum colored 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "433e7ac7d511768127ed85b0c4947f47a254131e37864b2dc13f52aa32cd37e5"
 "checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe"
 "checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59"
 "checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
@@ -3294,6 +3339,7 @@ dependencies = [
 "checksum deflate 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "707b6a7b384888a70c8d2e8650b3e60170dfc6a67bb4aa67b6dfca57af4bedb4"
 "checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe"
 "checksum deunicode 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690"
+"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
 "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
 "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
 "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
@@ -3375,6 +3421,7 @@ dependencies = [
 "checksum mio-extras 2.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19"
 "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
 "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
+"checksum mockito 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e524e85ea7c80559354217a6470c14abc2411802a9996aed1821559b9e28e3c"
 "checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e"
 "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
 "checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30"

components/link_checker/Cargo.toml

@@ -10,3 +10,6 @@ lazy_static = "1"
 
 config = { path = "../config" }
 errors = { path = "../errors" }
+
+[dev-dependencies]
+mockito = "0.22"

components/link_checker/src/lib.rs

@@ -140,20 +140,51 @@ fn check_page_for_anchor(url: &str, body: reqwest::Result<String>) -> Result<()>
 #[cfg(test)]
 mod tests {
     use super::{check_page_for_anchor, check_url, has_anchor, LinkChecker, LINKS};
+    use mockito::mock;
 
     #[test]
     fn can_validate_ok_links() {
-        let url = "https://google.com";
+        let url = format!("{}{}", mockito::server_url(), "/test");
-        let res = check_url(url, &LinkChecker::default());
+        let _m = mock("GET", "/test")
-        assert!(res.is_valid());
+            .with_header("content-type", "text/html")
-        assert!(LINKS.read().unwrap().get(url).is_some());
+            .with_body(format!(
-        let res = check_url(url, &LinkChecker::default());
+                r#"<!DOCTYPE html>
+<html>
+<head>
+  <title>Test</title>
+</head>
+<body>
+  <a href="{}">Mock URL</a>
+</body>
+</html>
+"#,
+                url
+            ))
+            .create();
+
+        let res = check_url(&url, &LinkChecker::default());
         assert!(res.is_valid());
+        assert!(LINKS.read().unwrap().get(&url).is_some());
+    }
+
+    #[test]
+    fn can_fail_unresolved_links() {
+        let res = check_url("https://google.comys", &LinkChecker::default());
+        assert_eq!(res.is_valid(), false);
+        assert!(res.code.is_none());
+        assert!(res.error.is_some());
     }
 
     #[test]
     fn can_fail_404_links() {
-        let res = check_url("https://google.comys", &LinkChecker::default());
+        let _m = mock("GET", "/404")
+            .with_status(404)
+            .with_header("content-type", "text/plain")
+            .with_body("Not Found")
+            .create();
+
+        let url = format!("{}{}", mockito::server_url(), "/404");
+        let res = check_url(&url, &LinkChecker::default());
         assert_eq!(res.is_valid(), false);
         assert!(res.code.is_none());
         assert!(res.error.is_some());
@@ -162,7 +193,7 @@ mod tests {
     #[test]
     fn can_validate_anchors() {
         let url = "https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.collect";
-        let body = "<body><h3 id='method.collect'>collect</h3></body>".to_string();
+        let body = r#"<body><h3 id="method.collect">collect</h3></body>"#.to_string();
         let res = check_page_for_anchor(url, Ok(body));
         assert!(res.is_ok());
     }
@@ -186,7 +217,7 @@ mod tests {
     #[test]
     fn can_fail_when_anchor_not_found() {
         let url = "https://doc.rust-lang.org/std/iter/trait.Iterator.html#me";
-        let body = "<body><h3 id='method.collect'>collect</h3></body>".to_string();
+        let body = r#"<body><h3 id="method.collect">collect</h3></body>"#.to_string();
         let res = check_page_for_anchor(url, Ok(body));
         assert!(res.is_err());
     }
@@ -221,21 +252,50 @@ mod tests {
 
     #[test]
     fn skip_anchor_prefixes() {
-        let config = LinkChecker {
+        let ignore_url = format!("{}{}", mockito::server_url(), "/ignore/");
-            skip_prefixes: vec![],
+        let config = LinkChecker { skip_prefixes: vec![], skip_anchor_prefixes: vec![ignore_url] };
-            skip_anchor_prefixes: vec!["https://github.com/rust-lang/rust/blob/".to_owned()],
+
-        };
+        let _m1 = mock("GET", "/ignore/test")
+            .with_header("content-type", "text/html")
+            .with_body(
+                r#"<!DOCTYPE html>
+<html>
+<head>
+  <title>Ignore</title>
+</head>
+<body>
+  <p id="existent"></p>
+</body>
+</html>
+"#,
+            )
+            .create();
 
         // anchor check is ignored because the url matches the prefix
-        let permalink = "https://github.com/rust-lang/rust/blob/c772948b687488a087356cb91432425662e034b9/src/librustc_back/target/mod.rs#L194-L214";
+        let ignore = format!("{}{}", mockito::server_url(), "/ignore/test#nonexistent");
-        assert!(check_url(&permalink, &config).is_valid());
+        assert!(check_url(&ignore, &config).is_valid());
+
+        let _m2 = mock("GET", "/test")
+            .with_header("content-type", "text/html")
+            .with_body(
+                r#"<!DOCTYPE html>
+<html>
+<head>
+  <title>Test</title>
+</head>
+<body>
+  <p id="existent"></p>
+</body>
+</html>
+"#,
+            )
+            .create();
 
         // other anchors are checked
-        let glossary = "https://help.github.com/en/articles/github-glossary#blame";
+        let existent = format!("{}{}", mockito::server_url(), "/test#existent");
-        assert!(check_url(&glossary, &config).is_valid());
+        assert!(check_url(&existent, &config).is_valid());
 
-        let glossary_invalid =
+        let nonexistent = format!("{}{}", mockito::server_url(), "/test#nonexistent");
-            "https://help.github.com/en/articles/github-glossary#anchor-does-not-exist";
+        assert_eq!(check_url(&nonexistent, &config).is_valid(), false);
-        assert_eq!(check_url(&glossary_invalid, &config).is_valid(), false);
     }
 }

components/templates/Cargo.toml

@@ -21,3 +21,6 @@ utils = { path = "../utils" }
 library = { path = "../library" }
 config = { path = "../config" }
 imageproc = { path = "../imageproc" }
+
+[dev-dependencies]
+mockito = "0.22"

components/templates/src/global_fns/load_data.rs

@@ -321,6 +321,7 @@ mod tests {
     use std::collections::HashMap;
     use std::path::PathBuf;
 
+    use mockito::mock;
     use serde_json::json;
     use tera::{to_value, Function};
 
@@ -365,10 +366,14 @@ mod tests {
 
     #[test]
     fn calculates_cache_key_for_url() {
-        let cache_key =
+        let _m = mock("GET", "/test")
-            DataSource::Url("https://api.github.com/repos/getzola/zola".parse().unwrap())
+            .with_header("content-type", "text/plain")
-                .get_cache_key(&OutputFormat::Plain);
+            .with_body("Test")
-        assert_eq!(cache_key, 8916756616423791754);
+            .create();
+
+        let url = format!("{}{}", mockito::server_url(), "/test");
+        let cache_key = DataSource::Url(url.parse().unwrap()).get_cache_key(&OutputFormat::Plain);
+        assert_eq!(cache_key, 12502656262443320092);
     }
 
     #[test]
@@ -391,28 +396,45 @@ mod tests {
 
     #[test]
     fn can_load_remote_data() {
+        let _m = mock("GET", "/json")
+            .with_header("content-type", "application/json")
+            .with_body(
+                r#"{
+  "test": {
+    "foo": "bar"
+  }
+}
+"#,
+            )
+            .create();
+
+        let url = format!("{}{}", mockito::server_url(), "/json");
         let static_fn = LoadData::new(PathBuf::new());
         let mut args = HashMap::new();
-        args.insert("url".to_string(), to_value("https://httpbin.org/json").unwrap());
+        args.insert("url".to_string(), to_value(&url).unwrap());
         args.insert("format".to_string(), to_value("json").unwrap());
         let result = static_fn.call(&args).unwrap();
-        assert_eq!(
+        assert_eq!(result.get("test").unwrap().get("foo").unwrap(), &to_value("bar").unwrap());
-            result.get("slideshow").unwrap().get("title").unwrap(),
-            &to_value("Sample Slide Show").unwrap()
-        );
     }
 
     #[test]
     fn fails_when_request_404s() {
+        let _m = mock("GET", "/404")
+            .with_status(404)
+            .with_header("content-type", "text/plain")
+            .with_body("Not Found")
+            .create();
+
+        let url = format!("{}{}", mockito::server_url(), "/404");
         let static_fn = LoadData::new(PathBuf::new());
         let mut args = HashMap::new();
-        args.insert("url".to_string(), to_value("https://httpbin.org/status/404/").unwrap());
+        args.insert("url".to_string(), to_value(&url).unwrap());
         args.insert("format".to_string(), to_value("json").unwrap());
         let result = static_fn.call(&args);
         assert!(result.is_err());
         assert_eq!(
             result.unwrap_err().to_string(),
-            "Failed to request https://httpbin.org/status/404/: 404 Not Found"
+            format!("Failed to request {}: 404 Not Found", url)
         );
     }