emelie/zola
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add HTTP POST capability to load_data() (#1400)

Browse source 
* Fixed failing tests on windows when user is not VssAdministrator.

* Fixed windows specific testcases related to \r

* Added the ability to perform POST requests to load_data

* make tests on windows deal with both \r being there on windows, and \r not being generated as on my personal windows system.

* undo earlier commit eaaa8c3ddd65d474161073a6fb80599eea1a9a21

because it fails on azure buildserver

* added new arguments to the hash for the cache function.

So caching now works as it should

* added new arguments to the hash for the cache function.

* improved documentation of load_data POST with better example.

* added basic derive traits

* changed load_data param contenttype to content_type

* fixed caching issues that went missing?

* format

* made code more idiomatic as suggested by keats
This commit is contained in:
Marco Tolk 2021-04-21 21:29:47 +02:00 committed by GitHub
parent 47b920777a
commit 1bd777f0e5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 322 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

325
components/templates/src/global_fns/load_data.rs
View file

@ -1,6 +1,7 @@
use utils::de::fix_toml_dates;
use utils::fs::{get_file_time, is_path_in_directory, read_file};
use reqwest::header::{HeaderValue, CONTENT_TYPE};
use reqwest::{blocking::Client, header};
use std::collections::hash_map::DefaultHasher;
use std::fmt;
@ -23,6 +24,24 @@ enum DataSource {
Path(PathBuf),
}
#[derive(Debug, PartialEq, Clone, Copy, Hash)]
enum Method {
Post,
Get,
}
impl FromStr for Method {
type Err = Error;
fn from_str(s: &str) -> Result<Self> {
match s.to_lowercase().as_ref() {
"post" => Ok(Method::Post),
"get" => Ok(Method::Get),
_ => Err("`load_data` method must either be POST or GET.".into()),
}
}
}
#[derive(Debug)]
enum OutputFormat {
Toml,
@ -104,9 +123,18 @@ impl DataSource {
Err(GET_DATA_ARGUMENT_ERROR_MESSAGE.into())
}
fn get_cache_key(&self, format: &OutputFormat) -> u64 {
fn get_cache_key(
&self,
format: &OutputFormat,
method_arg: Option<String>,
method_post_body: Option<String>,
method_post_contenttype: Option<String>,
) -> u64 {
let mut hasher = DefaultHasher::new();
format.hash(&mut hasher);
method_arg.hash(&mut hasher);
method_post_body.hash(&mut hasher);
method_post_contenttype.hash(&mut hasher);
self.hash(&mut hasher);
hasher.finish()
}
@ -201,6 +229,25 @@ impl TeraFn for LoadData {
};
let path_arg = optional_arg!(String, args.get("path"), GET_DATA_ARGUMENT_ERROR_MESSAGE);
let url_arg = optional_arg!(String, args.get("url"), GET_DATA_ARGUMENT_ERROR_MESSAGE);
let post_body_arg =
optional_arg!(String, args.get("body"), "`load_data` body must be a string, if set.");
let post_content_type = optional_arg!(
String,
args.get("content_type"),
"`load_data` content_type must be a string, if set."
);
let method_arg = optional_arg!(
String,
args.get("method"),
"`load_data` method must either be POST or GET."
);
let method = match method_arg {
Some(ref method_str) => match Method::from_str(&method_str) {
Ok(m) => m,
Err(e) => return Err(e),
},
_ => Method::Get,
};
let data_source = DataSource::from_args(path_arg.clone(), url_arg, &self.base_path)?;
// If the file doesn't exist, source is None
@ -222,7 +269,12 @@ impl TeraFn for LoadData {
}
let data_source = data_source.unwrap();
let file_format = get_output_format_from_args(&args, &data_source)?;
let cache_key = data_source.get_cache_key(&file_format);
let cache_key = data_source.get_cache_key(
&file_format,
method_arg,
post_body_arg.clone(),
post_content_type.clone(),
);
let mut cache = self.result_cache.lock().expect("result cache lock");
let response_client = self.client.lock().expect("response client lock");
@ -233,12 +285,40 @@ impl TeraFn for LoadData {
let data = match data_source {
DataSource::Path(path) => read_data_file(&self.base_path, path),
DataSource::Url(url) => {
match response_client
.get(url.as_str())
.header(header::ACCEPT, file_format.as_accept_header())
.send()
.and_then(|res| res.error_for_status())
{
let req = match method {
Method::Get => response_client
.get(url.as_str())
.header(header::ACCEPT, file_format.as_accept_header()),
Method::Post => {
let mut resp = response_client
.post(url.as_str())
.header(header::ACCEPT, file_format.as_accept_header());
match post_content_type {
Some(content_type) => match HeaderValue::from_str(&content_type) {
Ok(c) => {
resp = resp.header(CONTENT_TYPE, c);
}
Err(_) => {
return Err(format!(
"{} is an illegal contenttype",
&content_type
)
.into());
}
},
_ => {}
};
match post_body_arg {
Some(body) => {
resp = resp.body(body);
}
_ => {}
};
resp
}
};
match req.send().and_then(|res| res.error_for_status()) {
Ok(r) => r.text().map_err(|e| {
format!("Failed to parse response from {}: {:?}", url, e).into()
}),
@ -418,6 +498,106 @@ mod tests {
return test_files.join(filename);
}
#[test]
fn fails_illegal_method_parameter() {
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new();
args.insert("url".to_string(), to_value("https://example.com").unwrap());
args.insert("format".to_string(), to_value("plain").unwrap());
args.insert("method".to_string(), to_value("illegalmethod").unwrap());
let result = static_fn.call(&args);
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("`load_data` method must either be POST or GET."));
}
#[test]
fn can_load_remote_data_using_post_method() {
let _mg = mock("GET", "/kr1zdgbm4y")
.with_header("content-type", "text/plain")
.with_body("GET response")
.expect(0)
.create();
let _mp = mock("POST", "/kr1zdgbm4y")
.with_header("content-type", "text/plain")
.with_body("POST response")
.create();
let url = format!("{}{}", mockito::server_url(), "/kr1zdgbm4y");
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new();
args.insert("url".to_string(), to_value(url).unwrap());
args.insert("format".to_string(), to_value("plain").unwrap());
args.insert("method".to_string(), to_value("post").unwrap());
let result = static_fn.call(&args);
assert!(result.is_ok());
assert_eq!(result.unwrap(), "POST response");
_mg.assert();
_mp.assert();
}
#[test]
fn can_load_remote_data_using_post_method_with_content_type_header() {
let _mjson = mock("POST", "/kr1zdgbm4yw")
.match_header("content-type", "application/json")
.with_header("content-type", "application/json")
.with_body("{i_am:'json'}")
.expect(0)
.create();
let _mtext = mock("POST", "/kr1zdgbm4yw")
.match_header("content-type", "text/plain")
.with_header("content-type", "text/plain")
.with_body("POST response text")
.create();
let url = format!("{}{}", mockito::server_url(), "/kr1zdgbm4yw");
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new();
args.insert("url".to_string(), to_value(url).unwrap());
args.insert("format".to_string(), to_value("plain").unwrap());
args.insert("method".to_string(), to_value("post").unwrap());
args.insert("content_type".to_string(), to_value("text/plain").unwrap());
let result = static_fn.call(&args);
assert!(result.is_ok());
assert_eq!(result.unwrap(), "POST response text");
_mjson.assert();
_mtext.assert();
}
#[test]
fn can_load_remote_data_using_post_method_with_body() {
let _mjson = mock("POST", "/kr1zdgbm4y")
.match_body("qwerty")
.with_header("content-type", "application/json")
.with_body("{i_am:'json'}")
.expect(0)
.create();
let _mtext = mock("POST", "/kr1zdgbm4y")
.match_body("this is a match")
.with_header("content-type", "text/plain")
.with_body("POST response text")
.create();
let url = format!("{}{}", mockito::server_url(), "/kr1zdgbm4y");
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new();
args.insert("url".to_string(), to_value(url).unwrap());
args.insert("format".to_string(), to_value("plain").unwrap());
args.insert("method".to_string(), to_value("post").unwrap());
args.insert("content_type".to_string(), to_value("text/plain").unwrap());
args.insert("body".to_string(), to_value("this is a match").unwrap());
let result = static_fn.call(&args);
assert!(result.is_ok());
assert_eq!(result.unwrap(), "POST response text");
_mjson.assert();
_mtext.assert();
}
#[test]
fn fails_when_missing_file() {
let static_fn = LoadData::new(PathBuf::from("../utils"));
@ -456,10 +636,18 @@ mod tests {
#[test]
fn calculates_cache_key_for_path() {
// We can't test against a fixed value, due to the fact the cache key is built from the absolute path
let cache_key =
DataSource::Path(get_test_file("test.toml")).get_cache_key(&OutputFormat::Toml);
let cache_key_2 =
DataSource::Path(get_test_file("test.toml")).get_cache_key(&OutputFormat::Toml);
let cache_key = DataSource::Path(get_test_file("test.toml")).get_cache_key(
&OutputFormat::Toml,
None,
None,
None,
);
let cache_key_2 = DataSource::Path(get_test_file("test.toml")).get_cache_key(
&OutputFormat::Toml,
None,
None,
None,
);
assert_eq!(cache_key, cache_key_2);
}
@ -471,25 +659,46 @@ mod tests {
.create();
let url = format!("{}{}", mockito::server_url(), "/kr1zdgbm4y");
let cache_key = DataSource::Url(url.parse().unwrap()).get_cache_key(&OutputFormat::Plain);
assert_eq!(cache_key, 425638486551656875);
let cache_key = DataSource::Url(url.parse().unwrap()).get_cache_key(
&OutputFormat::Plain,
None,
None,
None,
);
assert_eq!(cache_key, 16044537454280534951);
}
#[test]
fn different_cache_key_per_filename() {
let toml_cache_key =
DataSource::Path(get_test_file("test.toml")).get_cache_key(&OutputFormat::Toml);
let json_cache_key =
DataSource::Path(get_test_file("test.json")).get_cache_key(&OutputFormat::Toml);
let toml_cache_key = DataSource::Path(get_test_file("test.toml")).get_cache_key(
&OutputFormat::Toml,
None,
None,
None,
);
let json_cache_key = DataSource::Path(get_test_file("test.json")).get_cache_key(
&OutputFormat::Toml,
None,
None,
None,
);
assert_ne!(toml_cache_key, json_cache_key);
}
#[test]
fn different_cache_key_per_format() {
let toml_cache_key =
DataSource::Path(get_test_file("test.toml")).get_cache_key(&OutputFormat::Toml);
let json_cache_key =
DataSource::Path(get_test_file("test.toml")).get_cache_key(&OutputFormat::Json);
let toml_cache_key = DataSource::Path(get_test_file("test.toml")).get_cache_key(
&OutputFormat::Toml,
None,
None,
None,
);
let json_cache_key = DataSource::Path(get_test_file("test.toml")).get_cache_key(
&OutputFormat::Json,
None,
None,
None,
);
assert_ne!(toml_cache_key, json_cache_key);
}
@ -609,7 +818,7 @@ mod tests {
let result = static_fn.call(&args.clone()).unwrap();
if cfg!(windows) {
assert_eq!(result, ".hello {}\r\n",);
assert_eq!(result.as_str().unwrap().replace("\r\n", "\n"), ".hello {}\n",);
} else {
assert_eq!(result, ".hello {}\n",);
};
@ -624,7 +833,10 @@ mod tests {
let result = static_fn.call(&args.clone()).unwrap();
if cfg!(windows) {
assert_eq!(result, "Number,Title\r\n1,Gutenberg\r\n2,Printing",);
assert_eq!(
result.as_str().unwrap().replace("\r\n", "\n"),
"Number,Title\n1,Gutenberg\n2,Printing",
);
} else {
assert_eq!(result, "Number,Title\n1,Gutenberg\n2,Printing",);
};
@ -639,7 +851,7 @@ mod tests {
let result = static_fn.call(&args.clone()).unwrap();
if cfg!(windows) {
assert_eq!(result, ".hello {}\r\n",);
assert_eq!(result.as_str().unwrap().replace("\r\n", "\n"), ".hello {}\n",);
} else {
assert_eq!(result, ".hello {}\n",);
};
@ -724,4 +936,67 @@ mod tests {
})
)
}
#[test]
fn is_load_remote_data_using_post_method_with_different_body_not_cached() {
let _mjson = mock("POST", "/kr1zdgbm4y3")
.with_header("content-type", "application/json")
.with_body("{i_am:'json'}")
.expect(2)
.create();
let url = format!("{}{}", mockito::server_url(), "/kr1zdgbm4y3");
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new();
args.insert("url".to_string(), to_value(&url).unwrap());
args.insert("format".to_string(), to_value("plain").unwrap());
args.insert("method".to_string(), to_value("post").unwrap());
args.insert("contenttype".to_string(), to_value("text/plain").unwrap());
args.insert("body".to_string(), to_value("this is a match").unwrap());
let result = static_fn.call(&args);
assert!(result.is_ok());
let mut args2 = HashMap::new();
args2.insert("url".to_string(), to_value(&url).unwrap());
args2.insert("format".to_string(), to_value("plain").unwrap());
args2.insert("method".to_string(), to_value("post").unwrap());
args2.insert("contenttype".to_string(), to_value("text/plain").unwrap());
args2.insert("body".to_string(), to_value("this is a match2").unwrap());
let result2 = static_fn.call(&args2);
assert!(result2.is_ok());
_mjson.assert();
}
#[test]
fn is_load_remote_data_using_post_method_with_same_body_cached() {
let _mjson = mock("POST", "/kr1zdgbm4y2")
.match_body("this is a match")
.with_header("content-type", "application/json")
.with_body("{i_am:'json'}")
.expect(1)
.create();
let url = format!("{}{}", mockito::server_url(), "/kr1zdgbm4y2");
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new();
args.insert("url".to_string(), to_value(&url).unwrap());
args.insert("format".to_string(), to_value("plain").unwrap());
args.insert("method".to_string(), to_value("post").unwrap());
args.insert("contenttype".to_string(), to_value("text/plain").unwrap());
args.insert("body".to_string(), to_value("this is a match").unwrap());
let result = static_fn.call(&args);
assert!(result.is_ok());
let mut args2 = HashMap::new();
args2.insert("url".to_string(), to_value(&url).unwrap());
args2.insert("format".to_string(), to_value("plain").unwrap());
args2.insert("method".to_string(), to_value("post").unwrap());
args2.insert("contenttype".to_string(), to_value("text/plain").unwrap());
args2.insert("body".to_string(), to_value("this is a match").unwrap());
let result2 = static_fn.call(&args2);
assert!(result2.is_ok());
_mjson.assert();
}
}

22
docs/content/documentation/templates/overview.md
View file

@ -339,6 +339,28 @@ as below.
{{ response }}
```
When no other parameters are specified the URL will always be retrieved using a HTTP GET request.
Using the parameter `method`, since version 0.14.0, you can also choose to retrieve the URL using a POST request.
When using `method="POST"` you can also use the parameters `body` and `contenttype`.
The parameter body is the actual contents sent in the POST request.
The parameter contenttype should be the mimetype of the body.
This example will make a POST request to the kroki service to generate a SVG.
```jinja2
{% set postdata = load_data(url="https://kroki.io/blockdiag/svg", format="plain", method="POST" ,content_type="text/plain", body="blockdiag {
'Doing POST' -> 'using load_data'
'using load_data' -> 'can generate' -> 'block diagrams';
'using load_data' -> is -> 'very easy!';
'Doing POST' [color = 'greenyellow'];
'block diagrams' [color = 'pink'];
'very easy!' [color = 'orange'];
}")%}
{{postdata|safe}}
```
#### Data caching
Data file loading and remote requests are cached in memory during the build, so multiple requests aren't made