From 998283d17cc330368dba11311cfbe9a8781779a0 Mon Sep 17 00:00:00 2001 From: Vincent Prouillet Date: Wed, 5 Jul 2017 08:27:27 +0900 Subject: [PATCH] Faster render_sitemap --- .gitignore | 15 +- Cargo.lock | 2 + components/site/Cargo.toml | 2 + {benches => components/site/benches}/gen.py | 0 components/site/benches/load.rs | 157 ++++++++++++++++++ components/site/benches/site.rs | 37 +++++ components/site/src/lib.rs | 36 +++- components/templates/src/builtins/sitemap.xml | 4 +- 8 files changed, 237 insertions(+), 16 deletions(-) rename {benches => components/site/benches}/gen.py (100%) create mode 100644 components/site/benches/load.rs create mode 100644 components/site/benches/site.rs diff --git a/.gitignore b/.gitignore index 8012b553..66e5644b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,13 @@ target .idea/ test_site/public -benches/small-blog -benches/medium-blog -benches/big-blog -benches/huge-blog -benches/small-kb -benches/medium-kb -benches/huge-kb +small-blog +medium-blog +big-blog +huge-blog +small-kb +medium-kb +huge-kb + current.bench now.bench diff --git a/Cargo.lock b/Cargo.lock index 151393f3..076c1b81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -872,6 +872,8 @@ dependencies = [ "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "pagination 0.1.0", "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "taxonomies 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "templates 0.1.0", diff --git a/components/site/Cargo.toml b/components/site/Cargo.toml index cd72a67f..5403d18a 100644 --- a/components/site/Cargo.toml +++ b/components/site/Cargo.toml @@ -8,6 +8,8 @@ tera = "0.10" glob = "0.2" walkdir = "1" rayon = "0.8" +serde = "1.0" +serde_derive = "1.0" errors = { path = "../errors" } config = { path = "../config" } diff --git a/benches/gen.py b/components/site/benches/gen.py similarity index 100% rename from benches/gen.py rename to components/site/benches/gen.py diff --git a/components/site/benches/load.rs b/components/site/benches/load.rs new file mode 100644 index 00000000..48ec3329 --- /dev/null +++ b/components/site/benches/load.rs @@ -0,0 +1,157 @@ +//! Benchmarking loading/markdown rendering of generated sites of various sizes + +#![feature(test)] +extern crate test; +extern crate site; + +use std::env; + +use site::Site; + + +#[bench] +fn bench_loading_small_blog(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("small-blog"); + let mut site = Site::new(&path, "config.toml").unwrap(); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_small_blog_with_syntax_highlighting(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("small-blog"); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.config.highlight_code = Some(true); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_medium_blog(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("medium-blog"); + let mut site = Site::new(&path, "config.toml").unwrap(); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("medium-blog"); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.config.highlight_code = Some(true); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_big_blog(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("big-blog"); + let mut site = Site::new(&path, "config.toml").unwrap(); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_big_blog_with_syntax_highlighting(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("big-blog"); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.config.highlight_code = Some(true); + + b.iter(|| site.load().unwrap()); +} + +//#[bench] +//fn bench_loading_huge_blog(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-blog"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// +// b.iter(|| site.load().unwrap()); +//} +// +//#[bench] +//fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-blog"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// site.config.highlight_code = Some(true); +// +// b.iter(|| site.load().unwrap()); +//} + +#[bench] +fn bench_loading_small_kb(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("small-kb"); + let mut site = Site::new(&path, "config.toml").unwrap(); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_small_kb_with_syntax_highlighting(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("small-kb"); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.config.highlight_code = Some(true); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_medium_kb(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("medium-kb"); + let mut site = Site::new(&path, "config.toml").unwrap(); + + b.iter(|| site.load().unwrap()); +} + +#[bench] +fn bench_loading_medium_kb_with_syntax_highlighting(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("medium-kb"); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.config.highlight_code = Some(true); + + b.iter(|| site.load().unwrap()); +} + +//#[bench] +//fn bench_loading_huge_kb(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-kb"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// +// b.iter(|| site.load().unwrap()); +//} +// +//#[bench] +//fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-kb"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// site.config.highlight_code = Some(true); +// +// b.iter(|| site.load().unwrap()); +//} diff --git a/components/site/benches/site.rs b/components/site/benches/site.rs new file mode 100644 index 00000000..3740fe50 --- /dev/null +++ b/components/site/benches/site.rs @@ -0,0 +1,37 @@ +#![feature(test)] +extern crate test; +extern crate site; +extern crate tempdir; + +use std::env; + +use tempdir::TempDir; +use site::Site; + + +fn setup_site(name: &str) -> Site { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push(name); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.load().unwrap(); + site +} + +#[bench] +fn bench_render_aliases(b: &mut test::Bencher) { + let mut site = setup_site("huge-blog"); + let tmp_dir = TempDir::new("benches").expect("create temp dir"); + let public = &tmp_dir.path().join("public"); + site.set_output_path(&public); + b.iter(|| site.render_aliases().unwrap()); +} + +#[bench] +fn bench_render_sitemap(b: &mut test::Bencher) { + let mut site = setup_site("huge-blog"); + let tmp_dir = TempDir::new("benches").expect("create temp dir"); + let public = &tmp_dir.path().join("public"); + site.set_output_path(&public); + b.iter(|| site.render_sitemap().unwrap()); +} diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index eb51d34e..285d2983 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -2,6 +2,9 @@ extern crate tera; extern crate rayon; extern crate glob; extern crate walkdir; +extern crate serde; +#[macro_use] +extern crate serde_derive; extern crate errors; extern crate config; @@ -35,6 +38,19 @@ use pagination::Paginator; use rayon::prelude::*; + +#[derive(Debug, Serialize)] +struct SitemapEntry { + permalink: String, + date: Option, +} + +impl SitemapEntry { + pub fn new(permalink: String, date: Option) -> SitemapEntry { + SitemapEntry { permalink, date } + } +} + #[derive(Debug)] pub struct Site { /// The base path of the gutenberg site @@ -406,7 +422,6 @@ impl Site { self.render_aliases()?; self.render_sections()?; self.render_orphan_pages()?; - // TODO: render_sitemap is slow self.render_sitemap()?; if self.config.generate_rss.unwrap() { self.render_rss_feed()?; @@ -496,16 +511,23 @@ impl Site { ensure_directory_exists(&self.output_path)?; let mut context = Context::new(); - context.add("pages", &self.pages.values().collect::>()); - context.add("sections", &self.sections.values().collect::>()); + + context.add( + "pages", + &self.pages.values().map(|p| SitemapEntry::new(p.permalink.clone(), p.meta.date.clone())).collect::>() + ); + context.add( + "sections", + &self.sections.values().map(|s| SitemapEntry::new(s.permalink.clone(), None)).collect::>() + ); let mut categories = vec![]; if let Some(ref c) = self.categories { let name = c.get_list_name(); - categories.push(self.config.make_permalink(&name)); + categories.push(SitemapEntry::new(self.config.make_permalink(&name), None)); for item in &c.items { categories.push( - self.config.make_permalink(&format!("{}/{}", &name, item.slug)) + SitemapEntry::new(self.config.make_permalink(&format!("{}/{}", &name, item.slug)), None), ); } } @@ -514,10 +536,10 @@ impl Site { let mut tags = vec![]; if let Some(ref t) = self.tags { let name = t.get_list_name(); - tags.push(self.config.make_permalink(&name)); + tags.push(SitemapEntry::new(self.config.make_permalink(&name), None)); for item in &t.items { tags.push( - self.config.make_permalink(&format!("{}/{}", &name, item.slug)) + SitemapEntry::new(self.config.make_permalink(&format!("{}/{}", &name, item.slug)), None), ); } } diff --git a/components/templates/src/builtins/sitemap.xml b/components/templates/src/builtins/sitemap.xml index d867752c..3ee6bbcb 100644 --- a/components/templates/src/builtins/sitemap.xml +++ b/components/templates/src/builtins/sitemap.xml @@ -14,12 +14,12 @@ {% endfor %} {% for category in categories %} - {{ category | safe }} + {{ category.permalink | safe }} {% endfor %} {% for tag in tags %} - {{ tag | safe }} + {{ tag.permalink | safe }} {% endfor %}