Faster render_sitemap

This commit is contained in:
Vincent Prouillet 2017-07-05 08:27:27 +09:00
parent ce704097a4
commit 998283d17c
8 changed files with 237 additions and 16 deletions

15
.gitignore vendored
View file

@ -2,12 +2,13 @@ target
.idea/ .idea/
test_site/public test_site/public
benches/small-blog small-blog
benches/medium-blog medium-blog
benches/big-blog big-blog
benches/huge-blog huge-blog
benches/small-kb small-kb
benches/medium-kb medium-kb
benches/huge-kb huge-kb
current.bench current.bench
now.bench now.bench

2
Cargo.lock generated
View file

@ -872,6 +872,8 @@ dependencies = [
"glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"pagination 0.1.0", "pagination 0.1.0",
"rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
"taxonomies 0.1.0", "taxonomies 0.1.0",
"tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"templates 0.1.0", "templates 0.1.0",

View file

@ -8,6 +8,8 @@ tera = "0.10"
glob = "0.2" glob = "0.2"
walkdir = "1" walkdir = "1"
rayon = "0.8" rayon = "0.8"
serde = "1.0"
serde_derive = "1.0"
errors = { path = "../errors" } errors = { path = "../errors" }
config = { path = "../config" } config = { path = "../config" }

View file

@ -0,0 +1,157 @@
//! Benchmarking loading/markdown rendering of generated sites of various sizes
#![feature(test)]
extern crate test;
extern crate site;
use std::env;
use site::Site;
#[bench]
fn bench_loading_small_blog(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("small-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_small_blog_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("small-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_medium_blog(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("medium-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("medium-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_big_blog(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("big-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_big_blog_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("big-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
//#[bench]
//fn bench_loading_huge_blog(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-blog");
// let mut site = Site::new(&path, "config.toml").unwrap();
//
// b.iter(|| site.load().unwrap());
//}
//
//#[bench]
//fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-blog");
// let mut site = Site::new(&path, "config.toml").unwrap();
// site.config.highlight_code = Some(true);
//
// b.iter(|| site.load().unwrap());
//}
#[bench]
fn bench_loading_small_kb(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("small-kb");
let mut site = Site::new(&path, "config.toml").unwrap();
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_small_kb_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("small-kb");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_medium_kb(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("medium-kb");
let mut site = Site::new(&path, "config.toml").unwrap();
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_medium_kb_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("medium-kb");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
//#[bench]
//fn bench_loading_huge_kb(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-kb");
// let mut site = Site::new(&path, "config.toml").unwrap();
//
// b.iter(|| site.load().unwrap());
//}
//
//#[bench]
//fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-kb");
// let mut site = Site::new(&path, "config.toml").unwrap();
// site.config.highlight_code = Some(true);
//
// b.iter(|| site.load().unwrap());
//}

View file

@ -0,0 +1,37 @@
#![feature(test)]
extern crate test;
extern crate site;
extern crate tempdir;
use std::env;
use tempdir::TempDir;
use site::Site;
fn setup_site(name: &str) -> Site {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push(name);
let mut site = Site::new(&path, "config.toml").unwrap();
site.load().unwrap();
site
}
#[bench]
fn bench_render_aliases(b: &mut test::Bencher) {
let mut site = setup_site("huge-blog");
let tmp_dir = TempDir::new("benches").expect("create temp dir");
let public = &tmp_dir.path().join("public");
site.set_output_path(&public);
b.iter(|| site.render_aliases().unwrap());
}
#[bench]
fn bench_render_sitemap(b: &mut test::Bencher) {
let mut site = setup_site("huge-blog");
let tmp_dir = TempDir::new("benches").expect("create temp dir");
let public = &tmp_dir.path().join("public");
site.set_output_path(&public);
b.iter(|| site.render_sitemap().unwrap());
}

View file

@ -2,6 +2,9 @@ extern crate tera;
extern crate rayon; extern crate rayon;
extern crate glob; extern crate glob;
extern crate walkdir; extern crate walkdir;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate errors; extern crate errors;
extern crate config; extern crate config;
@ -35,6 +38,19 @@ use pagination::Paginator;
use rayon::prelude::*; use rayon::prelude::*;
#[derive(Debug, Serialize)]
struct SitemapEntry {
permalink: String,
date: Option<String>,
}
impl SitemapEntry {
pub fn new(permalink: String, date: Option<String>) -> SitemapEntry {
SitemapEntry { permalink, date }
}
}
#[derive(Debug)] #[derive(Debug)]
pub struct Site { pub struct Site {
/// The base path of the gutenberg site /// The base path of the gutenberg site
@ -406,7 +422,6 @@ impl Site {
self.render_aliases()?; self.render_aliases()?;
self.render_sections()?; self.render_sections()?;
self.render_orphan_pages()?; self.render_orphan_pages()?;
// TODO: render_sitemap is slow
self.render_sitemap()?; self.render_sitemap()?;
if self.config.generate_rss.unwrap() { if self.config.generate_rss.unwrap() {
self.render_rss_feed()?; self.render_rss_feed()?;
@ -496,16 +511,23 @@ impl Site {
ensure_directory_exists(&self.output_path)?; ensure_directory_exists(&self.output_path)?;
let mut context = Context::new(); let mut context = Context::new();
context.add("pages", &self.pages.values().collect::<Vec<&Page>>());
context.add("sections", &self.sections.values().collect::<Vec<&Section>>()); context.add(
"pages",
&self.pages.values().map(|p| SitemapEntry::new(p.permalink.clone(), p.meta.date.clone())).collect::<Vec<_>>()
);
context.add(
"sections",
&self.sections.values().map(|s| SitemapEntry::new(s.permalink.clone(), None)).collect::<Vec<_>>()
);
let mut categories = vec![]; let mut categories = vec![];
if let Some(ref c) = self.categories { if let Some(ref c) = self.categories {
let name = c.get_list_name(); let name = c.get_list_name();
categories.push(self.config.make_permalink(&name)); categories.push(SitemapEntry::new(self.config.make_permalink(&name), None));
for item in &c.items { for item in &c.items {
categories.push( categories.push(
self.config.make_permalink(&format!("{}/{}", &name, item.slug)) SitemapEntry::new(self.config.make_permalink(&format!("{}/{}", &name, item.slug)), None),
); );
} }
} }
@ -514,10 +536,10 @@ impl Site {
let mut tags = vec![]; let mut tags = vec![];
if let Some(ref t) = self.tags { if let Some(ref t) = self.tags {
let name = t.get_list_name(); let name = t.get_list_name();
tags.push(self.config.make_permalink(&name)); tags.push(SitemapEntry::new(self.config.make_permalink(&name), None));
for item in &t.items { for item in &t.items {
tags.push( tags.push(
self.config.make_permalink(&format!("{}/{}", &name, item.slug)) SitemapEntry::new(self.config.make_permalink(&format!("{}/{}", &name, item.slug)), None),
); );
} }
} }

View file

@ -14,12 +14,12 @@
{% endfor %} {% endfor %}
{% for category in categories %} {% for category in categories %}
<url> <url>
<loc>{{ category | safe }}</loc> <loc>{{ category.permalink | safe }}</loc>
</url> </url>
{% endfor %} {% endfor %}
{% for tag in tags %} {% for tag in tags %}
<url> <url>
<loc>{{ tag | safe }}</loc> <loc>{{ tag.permalink | safe }}</loc>
</url> </url>
{% endfor %} {% endfor %}
</urlset> </urlset>