From b158ca79528928ab5ca176888686d3b078a7ad1c Mon Sep 17 00:00:00 2001 From: Vincent Prouillet Date: Thu, 22 Jun 2017 12:01:45 +0900 Subject: [PATCH] Read off disk in parallel --- CHANGELOG.md | 2 +- benches/{integration.rs => site_load.rs} | 104 +++++++++++------------ benches/unit.rs | 3 - src/bin/rebuild.rs | 6 +- src/site.rs | 60 ++++++++++--- 5 files changed, 102 insertions(+), 73 deletions(-) rename benches/{integration.rs => site_load.rs} (73%) delete mode 100644 benches/unit.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a809ca9e..3cdc3fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.0.8 (unreleased) -- Parallelize page rendering using rayon +- Parallelize all the things ## 0.0.7 (2017-06-19) diff --git a/benches/integration.rs b/benches/site_load.rs similarity index 73% rename from benches/integration.rs rename to benches/site_load.rs index 07036f25..9d8287d1 100644 --- a/benches/integration.rs +++ b/benches/site_load.rs @@ -1,13 +1,11 @@ -//! Benchmarking generated sites of various sizes +//! Benchmarking loading/markdown rendering of generated sites of various sizes #![feature(test)] extern crate test; extern crate gutenberg; -extern crate tempdir; use std::env; -use tempdir::TempDir; use gutenberg::Site; @@ -42,16 +40,16 @@ fn bench_loading_medium_blog(b: &mut test::Bencher) { b.iter(|| site.load().unwrap()); } -//#[bench] -//fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) { -// let mut path = env::current_dir().unwrap().to_path_buf(); -// path.push("benches"); -// path.push("medium-blog"); -// let mut site = Site::new(&path, "config.toml").unwrap(); -// site.config.highlight_code = Some(true); -// -// b.iter(|| site.load().unwrap()); -//} +#[bench] +fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) { + let mut path = env::current_dir().unwrap().to_path_buf(); + path.push("benches"); + path.push("medium-blog"); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.config.highlight_code = Some(true); + + b.iter(|| site.load().unwrap()); +} #[bench] fn bench_loading_big_blog(b: &mut test::Bencher) { @@ -74,26 +72,26 @@ fn bench_loading_big_blog_with_syntax_highlighting(b: &mut test::Bencher) { b.iter(|| site.load().unwrap()); } -#[bench] -fn bench_loading_huge_blog(b: &mut test::Bencher) { - let mut path = env::current_dir().unwrap().to_path_buf(); - path.push("benches"); - path.push("huge-blog"); - let mut site = Site::new(&path, "config.toml").unwrap(); - - b.iter(|| site.load().unwrap()); -} - -#[bench] -fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) { - let mut path = env::current_dir().unwrap().to_path_buf(); - path.push("benches"); - path.push("huge-blog"); - let mut site = Site::new(&path, "config.toml").unwrap(); - site.config.highlight_code = Some(true); - - b.iter(|| site.load().unwrap()); -} +//#[bench] +//fn bench_loading_huge_blog(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-blog"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// +// b.iter(|| site.load().unwrap()); +//} +// +//#[bench] +//fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-blog"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// site.config.highlight_code = Some(true); +// +// b.iter(|| site.load().unwrap()); +//} #[bench] fn bench_loading_small_kb(b: &mut test::Bencher) { @@ -137,23 +135,23 @@ fn bench_loading_medium_kb_with_syntax_highlighting(b: &mut test::Bencher) { b.iter(|| site.load().unwrap()); } -#[bench] -fn bench_loading_huge_kb(b: &mut test::Bencher) { - let mut path = env::current_dir().unwrap().to_path_buf(); - path.push("benches"); - path.push("huge-kb"); - let mut site = Site::new(&path, "config.toml").unwrap(); - - b.iter(|| site.load().unwrap()); -} - -#[bench] -fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) { - let mut path = env::current_dir().unwrap().to_path_buf(); - path.push("benches"); - path.push("huge-kb"); - let mut site = Site::new(&path, "config.toml").unwrap(); - site.config.highlight_code = Some(true); - - b.iter(|| site.load().unwrap()); -} +//#[bench] +//fn bench_loading_huge_kb(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-kb"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// +// b.iter(|| site.load().unwrap()); +//} +// +//#[bench] +//fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) { +// let mut path = env::current_dir().unwrap().to_path_buf(); +// path.push("benches"); +// path.push("huge-kb"); +// let mut site = Site::new(&path, "config.toml").unwrap(); +// site.config.highlight_code = Some(true); +// +// b.iter(|| site.load().unwrap()); +//} diff --git a/benches/unit.rs b/benches/unit.rs deleted file mode 100644 index b08b2290..00000000 --- a/benches/unit.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Benchmarking individual functions of Gutenberg - - diff --git a/src/bin/rebuild.rs b/src/bin/rebuild.rs index e52b915b..ecebc55b 100644 --- a/src/bin/rebuild.rs +++ b/src/bin/rebuild.rs @@ -130,7 +130,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> { // A section was edited if is_section { - match site.add_section(path, true)? { + let section = Section::from_file(path, &site.config)?; + match site.add_section(section, true)? { Some(prev) => { // Updating a section let current_meta = site.sections[path].meta.clone(); @@ -166,7 +167,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> { } // A page was edited - match site.add_page(path, true)? { + let page = Page::from_file(path, &site.config)?; + match site.add_page(page, true)? { Some(prev) => { site.register_get_url_fn(); // Updating a page diff --git a/src/site.rs b/src/site.rs index decae1b6..8cd62f5f 100644 --- a/src/site.rs +++ b/src/site.rs @@ -96,14 +96,46 @@ impl Site { let base_path = self.base_path.to_string_lossy().replace("\\", "/"); let content_glob = format!("{}/{}", base_path, "content/**/*.md"); - for entry in glob(&content_glob).unwrap().filter_map(|e| e.ok()) { - let path = entry.as_path(); - if path.file_name().unwrap() == "_index.md" { - self.add_section(path, false)?; - } else { - self.add_page(path, false)?; - } + let (section_entries, page_entries): (Vec<_>, Vec<_>) = glob(&content_glob) + .unwrap() + .filter_map(|e| e.ok()) + .partition(|ref entry| entry.as_path().file_name().unwrap() == "_index.md"); + + let sections = { + let config = &self.config; + + section_entries + .into_par_iter() + .filter(|entry| entry.as_path().file_name().unwrap() == "_index.md") + .map(|entry| { + let path = entry.as_path(); + Section::from_file(path, &config) + }).collect::>() + }; + + let pages = { + let config = &self.config; + + page_entries + .into_par_iter() + .filter(|entry| entry.as_path().file_name().unwrap() != "_index.md") + .map(|entry| { + let path = entry.as_path(); + Page::from_file(path, &config) + }).collect::>() + }; + + // Kinda duplicated code for add_section/add_page but necessary to do it that + // way because of the borrow checker + for section in sections { + let s = section?; + self.add_section(s, false)?; } + for page in pages { + let p = page?; + self.add_page(p, false)?; + } + // Insert a default index section if necessary so we don't need to create // a _index.md to render the index page let index_path = self.base_path.join("content").join("_index.md"); @@ -161,14 +193,14 @@ impl Site { /// The `render` parameter is used in the serve command, when rebuilding a page. /// If `true`, it will also render the markdown for that page /// Returns the previous page struct if there was one - pub fn add_page(&mut self, path: &Path, render: bool) -> Result> { - let page = Page::from_file(&path, &self.config)?; + pub fn add_page(&mut self, page: Page, render: bool) -> Result> { + let path = page.file.path.clone(); self.permalinks.insert(page.file.relative.clone(), page.permalink.clone()); let prev = self.pages.insert(page.file.path.clone(), page); if render { - let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[path].file.parent); - let mut page = self.pages.get_mut(path).unwrap(); + let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[&path].file.parent); + let mut page = self.pages.get_mut(&path).unwrap(); page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?; } @@ -179,13 +211,13 @@ impl Site { /// The `render` parameter is used in the serve command, when rebuilding a page. /// If `true`, it will also render the markdown for that page /// Returns the previous section struct if there was one - pub fn add_section(&mut self, path: &Path, render: bool) -> Result> { - let section = Section::from_file(path, &self.config)?; + pub fn add_section(&mut self, section: Section, render: bool) -> Result> { + let path = section.file.path.clone(); self.permalinks.insert(section.file.relative.clone(), section.permalink.clone()); let prev = self.sections.insert(section.file.path.clone(), section); if render { - let mut section = self.sections.get_mut(path).unwrap(); + let mut section = self.sections.get_mut(&path).unwrap(); section.render_markdown(&self.permalinks, &self.tera, &self.config)?; }