Read off disk in parallel

This commit is contained in:
Vincent Prouillet 2017-06-22 12:01:45 +09:00
parent b45f8c3116
commit b158ca7952
5 changed files with 102 additions and 73 deletions

View file

@ -2,7 +2,7 @@
## 0.0.8 (unreleased) ## 0.0.8 (unreleased)
- Parallelize page rendering using rayon - Parallelize all the things
## 0.0.7 (2017-06-19) ## 0.0.7 (2017-06-19)

View file

@ -1,13 +1,11 @@
//! Benchmarking generated sites of various sizes //! Benchmarking loading/markdown rendering of generated sites of various sizes
#![feature(test)] #![feature(test)]
extern crate test; extern crate test;
extern crate gutenberg; extern crate gutenberg;
extern crate tempdir;
use std::env; use std::env;
use tempdir::TempDir;
use gutenberg::Site; use gutenberg::Site;
@ -42,16 +40,16 @@ fn bench_loading_medium_blog(b: &mut test::Bencher) {
b.iter(|| site.load().unwrap()); b.iter(|| site.load().unwrap());
} }
//#[bench] #[bench]
//fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) { fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf(); let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches"); path.push("benches");
// path.push("medium-blog"); path.push("medium-blog");
// let mut site = Site::new(&path, "config.toml").unwrap(); let mut site = Site::new(&path, "config.toml").unwrap();
// site.config.highlight_code = Some(true); site.config.highlight_code = Some(true);
//
// b.iter(|| site.load().unwrap()); b.iter(|| site.load().unwrap());
//} }
#[bench] #[bench]
fn bench_loading_big_blog(b: &mut test::Bencher) { fn bench_loading_big_blog(b: &mut test::Bencher) {
@ -74,26 +72,26 @@ fn bench_loading_big_blog_with_syntax_highlighting(b: &mut test::Bencher) {
b.iter(|| site.load().unwrap()); b.iter(|| site.load().unwrap());
} }
#[bench] //#[bench]
fn bench_loading_huge_blog(b: &mut test::Bencher) { //fn bench_loading_huge_blog(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf(); // let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches"); // path.push("benches");
path.push("huge-blog"); // path.push("huge-blog");
let mut site = Site::new(&path, "config.toml").unwrap(); // let mut site = Site::new(&path, "config.toml").unwrap();
//
b.iter(|| site.load().unwrap()); // b.iter(|| site.load().unwrap());
} //}
//
#[bench] //#[bench]
fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) { //fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf(); // let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches"); // path.push("benches");
path.push("huge-blog"); // path.push("huge-blog");
let mut site = Site::new(&path, "config.toml").unwrap(); // let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true); // site.config.highlight_code = Some(true);
//
b.iter(|| site.load().unwrap()); // b.iter(|| site.load().unwrap());
} //}
#[bench] #[bench]
fn bench_loading_small_kb(b: &mut test::Bencher) { fn bench_loading_small_kb(b: &mut test::Bencher) {
@ -137,23 +135,23 @@ fn bench_loading_medium_kb_with_syntax_highlighting(b: &mut test::Bencher) {
b.iter(|| site.load().unwrap()); b.iter(|| site.load().unwrap());
} }
#[bench] //#[bench]
fn bench_loading_huge_kb(b: &mut test::Bencher) { //fn bench_loading_huge_kb(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf(); // let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches"); // path.push("benches");
path.push("huge-kb"); // path.push("huge-kb");
let mut site = Site::new(&path, "config.toml").unwrap(); // let mut site = Site::new(&path, "config.toml").unwrap();
//
b.iter(|| site.load().unwrap()); // b.iter(|| site.load().unwrap());
} //}
//
#[bench] //#[bench]
fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) { //fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf(); // let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches"); // path.push("benches");
path.push("huge-kb"); // path.push("huge-kb");
let mut site = Site::new(&path, "config.toml").unwrap(); // let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true); // site.config.highlight_code = Some(true);
//
b.iter(|| site.load().unwrap()); // b.iter(|| site.load().unwrap());
} //}

View file

@ -1,3 +0,0 @@
//! Benchmarking individual functions of Gutenberg

View file

@ -130,7 +130,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> {
// A section was edited // A section was edited
if is_section { if is_section {
match site.add_section(path, true)? { let section = Section::from_file(path, &site.config)?;
match site.add_section(section, true)? {
Some(prev) => { Some(prev) => {
// Updating a section // Updating a section
let current_meta = site.sections[path].meta.clone(); let current_meta = site.sections[path].meta.clone();
@ -166,7 +167,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> {
} }
// A page was edited // A page was edited
match site.add_page(path, true)? { let page = Page::from_file(path, &site.config)?;
match site.add_page(page, true)? {
Some(prev) => { Some(prev) => {
site.register_get_url_fn(); site.register_get_url_fn();
// Updating a page // Updating a page

View file

@ -96,14 +96,46 @@ impl Site {
let base_path = self.base_path.to_string_lossy().replace("\\", "/"); let base_path = self.base_path.to_string_lossy().replace("\\", "/");
let content_glob = format!("{}/{}", base_path, "content/**/*.md"); let content_glob = format!("{}/{}", base_path, "content/**/*.md");
for entry in glob(&content_glob).unwrap().filter_map(|e| e.ok()) { let (section_entries, page_entries): (Vec<_>, Vec<_>) = glob(&content_glob)
.unwrap()
.filter_map(|e| e.ok())
.partition(|ref entry| entry.as_path().file_name().unwrap() == "_index.md");
let sections = {
let config = &self.config;
section_entries
.into_par_iter()
.filter(|entry| entry.as_path().file_name().unwrap() == "_index.md")
.map(|entry| {
let path = entry.as_path(); let path = entry.as_path();
if path.file_name().unwrap() == "_index.md" { Section::from_file(path, &config)
self.add_section(path, false)?; }).collect::<Vec<_>>()
} else { };
self.add_page(path, false)?;
let pages = {
let config = &self.config;
page_entries
.into_par_iter()
.filter(|entry| entry.as_path().file_name().unwrap() != "_index.md")
.map(|entry| {
let path = entry.as_path();
Page::from_file(path, &config)
}).collect::<Vec<_>>()
};
// Kinda duplicated code for add_section/add_page but necessary to do it that
// way because of the borrow checker
for section in sections {
let s = section?;
self.add_section(s, false)?;
} }
for page in pages {
let p = page?;
self.add_page(p, false)?;
} }
// Insert a default index section if necessary so we don't need to create // Insert a default index section if necessary so we don't need to create
// a _index.md to render the index page // a _index.md to render the index page
let index_path = self.base_path.join("content").join("_index.md"); let index_path = self.base_path.join("content").join("_index.md");
@ -161,14 +193,14 @@ impl Site {
/// The `render` parameter is used in the serve command, when rebuilding a page. /// The `render` parameter is used in the serve command, when rebuilding a page.
/// If `true`, it will also render the markdown for that page /// If `true`, it will also render the markdown for that page
/// Returns the previous page struct if there was one /// Returns the previous page struct if there was one
pub fn add_page(&mut self, path: &Path, render: bool) -> Result<Option<Page>> { pub fn add_page(&mut self, page: Page, render: bool) -> Result<Option<Page>> {
let page = Page::from_file(&path, &self.config)?; let path = page.file.path.clone();
self.permalinks.insert(page.file.relative.clone(), page.permalink.clone()); self.permalinks.insert(page.file.relative.clone(), page.permalink.clone());
let prev = self.pages.insert(page.file.path.clone(), page); let prev = self.pages.insert(page.file.path.clone(), page);
if render { if render {
let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[path].file.parent); let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[&path].file.parent);
let mut page = self.pages.get_mut(path).unwrap(); let mut page = self.pages.get_mut(&path).unwrap();
page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?; page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
} }
@ -179,13 +211,13 @@ impl Site {
/// The `render` parameter is used in the serve command, when rebuilding a page. /// The `render` parameter is used in the serve command, when rebuilding a page.
/// If `true`, it will also render the markdown for that page /// If `true`, it will also render the markdown for that page
/// Returns the previous section struct if there was one /// Returns the previous section struct if there was one
pub fn add_section(&mut self, path: &Path, render: bool) -> Result<Option<Section>> { pub fn add_section(&mut self, section: Section, render: bool) -> Result<Option<Section>> {
let section = Section::from_file(path, &self.config)?; let path = section.file.path.clone();
self.permalinks.insert(section.file.relative.clone(), section.permalink.clone()); self.permalinks.insert(section.file.relative.clone(), section.permalink.clone());
let prev = self.sections.insert(section.file.path.clone(), section); let prev = self.sections.insert(section.file.path.clone(), section);
if render { if render {
let mut section = self.sections.get_mut(path).unwrap(); let mut section = self.sections.get_mut(&path).unwrap();
section.render_markdown(&self.permalinks, &self.tera, &self.config)?; section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
} }