From 6a5ace62fcfce45a8e7ea3b2006577e24d010371 Mon Sep 17 00:00:00 2001
From: Thomas Hurst <tom@hur.st>
Date: Sat, 25 Aug 2018 16:50:01 +0100
Subject: [PATCH] link_checker: Set Accept header

As mentioned in #381, crates.io 404's any request without an Accept:
text/html header.  It 200's any request with one, but at least
false-successes don't prevent checking any other links.

This also makes it easier to add a custom User-Agent if desired.

rustfmt and fix a clippy nit (unnecessary return) while I'm here.
---
 components/link_checker/src/lib.rs | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/components/link_checker/src/lib.rs b/components/link_checker/src/lib.rs
index 4aa2d816..1aac769c 100644
--- a/components/link_checker/src/lib.rs
+++ b/components/link_checker/src/lib.rs
@@ -2,11 +2,11 @@ extern crate reqwest;
 #[macro_use]
 extern crate lazy_static;
 
+use reqwest::header::{qitem, Accept, Headers};
+use reqwest::{mime, StatusCode};
 use std::collections::HashMap;
 use std::error::Error;
 use std::sync::{Arc, RwLock};
-use reqwest::StatusCode;
-
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct LinkResult {
@@ -54,19 +54,30 @@ pub fn check_url(url: &str) -> LinkResult {
         }
     }
 
+    let mut headers = Headers::new();
+    headers.set(Accept(vec![qitem(mime::TEXT_HTML), qitem(mime::STAR_STAR)]));
+
+    let client = reqwest::Client::new();
+
     // Need to actually do the link checking
-    let res = match reqwest::get(url) {
-        Ok(response) => LinkResult { code: Some(response.status()), error: None },
-        Err(e) => LinkResult { code: None, error: Some(e.description().to_string()) },
+    let res = match client.get(url).headers(headers).send() {
+        Ok(response) => LinkResult {
+            code: Some(response.status()),
+            error: None,
+        },
+        Err(e) => LinkResult {
+            code: None,
+            error: Some(e.description().to_string()),
+        },
     };
 
     LINKS.write().unwrap().insert(url.to_string(), res.clone());
-    return res;
+    res
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{LINKS, check_url};
+    use super::{check_url, LINKS};
 
     #[test]
     fn can_validate_ok_links() {