From 09d5e74a653b4f3d4825791a667892c90e3ffbf2 Mon Sep 17 00:00:00 2001 From: Vincent Prouillet Date: Sat, 13 May 2017 22:37:01 +0900 Subject: [PATCH] Smarter rebuild on content change Fix #59 --- CHANGELOG.md | 5 +- src/bin/cmd/serve.rs | 5 +- src/bin/gutenberg.rs | 1 + src/bin/rebuild.rs | 230 ++++++++++++++++++++++++++++++++++++ src/front_matter/page.rs | 17 ++- src/page.rs | 9 +- src/section.rs | 17 +++ src/site.rs | 249 ++++++++++++++++++--------------------- 8 files changed, 386 insertions(+), 147 deletions(-) create mode 100644 src/bin/rebuild.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 72a08eb0..9d112fd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,14 @@ - Fix XML templates overriding and reloading - `title` and `description` are now optional in the front matter -- Add GenericConfig, Vim syntax +- Add GenericConfig, Vim, Jinja2 syntax - Add `_index.md` for homepage as well and make that into a normal section - Allow sorting by `none`, `date` and `order` for sections - Add pagination - Add a `get_page` global function to tera +- Revamp index page, no more `pages` variables +- Fix livereload stopping randomly +- Smarter re-rendering in `serve` command ## 0.0.4 (2017-04-23) diff --git a/src/bin/cmd/serve.rs b/src/bin/cmd/serve.rs index 9c3e822d..9333055a 100644 --- a/src/bin/cmd/serve.rs +++ b/src/bin/cmd/serve.rs @@ -14,6 +14,7 @@ use gutenberg::Site; use gutenberg::errors::{Result, ResultExt}; use console; +use rebuild; #[derive(Debug, PartialEq)] enum ChangeKind { @@ -137,12 +138,12 @@ pub fn serve(interface: &str, port: &str, config_file: &str) -> Result<()> { (ChangeKind::Content, _) => { console::info(&format!("-> Content changed {}", path.display())); // Force refresh - rebuild_done_handling(&broadcaster, site.rebuild_after_content_change(&path), "/x.js"); + rebuild_done_handling(&broadcaster, rebuild::after_content_change(&mut site, &path), "/x.js"); }, (ChangeKind::Templates, _) => { console::info(&format!("-> Template changed {}", path.display())); // Force refresh - rebuild_done_handling(&broadcaster, site.rebuild_after_template_change(&path), "/x.js"); + rebuild_done_handling(&broadcaster, rebuild::after_template_change(&mut site, &path), "/x.js"); }, (ChangeKind::StaticFiles, p) => { if path.is_file() { diff --git a/src/bin/gutenberg.rs b/src/bin/gutenberg.rs index 2524990f..516e5ff4 100644 --- a/src/bin/gutenberg.rs +++ b/src/bin/gutenberg.rs @@ -16,6 +16,7 @@ use std::time::Instant; mod cmd; mod console; +mod rebuild; fn main() { diff --git a/src/bin/rebuild.rs b/src/bin/rebuild.rs new file mode 100644 index 00000000..2273f1c1 --- /dev/null +++ b/src/bin/rebuild.rs @@ -0,0 +1,230 @@ +use std::path::Path; + +use gutenberg::{Site, SectionFrontMatter, PageFrontMatter}; +use gutenberg::errors::Result; + +#[derive(Debug, Clone, Copy, PartialEq)] +enum PageChangesNeeded { + /// Editing `tags` + Tags, + /// Editing `categories` + Categories, + /// Editing `date` or `order` + Sort, + /// Editing anything else + Render, +} + +// TODO: seems like editing sort_by/render do weird stuff +#[derive(Debug, Clone, Copy, PartialEq)] +enum SectionChangesNeeded { + /// Editing `sort_by` + Sort, + /// Editing `title`, `description`, `extra`, `template` or setting `render` to true + Render, + /// Editing `paginate_by` or `paginate_path` + RenderWithPages, + /// Setting `render` to false + Delete, +} + +/// Evaluates all the params in the front matter that changed so we can do the smallest +/// delta in the serve command +fn find_section_front_matter_changes(current: &SectionFrontMatter, other: &SectionFrontMatter) -> Vec { + let mut changes_needed = vec![]; + + if current.sort_by != other.sort_by { + changes_needed.push(SectionChangesNeeded::Sort); + } + + if !current.should_render() && other.should_render() { + changes_needed.push(SectionChangesNeeded::Delete); + // Nothing else we can do + return changes_needed; + } + + if current.paginate_by != other.paginate_by || current.paginate_path != other.paginate_path { + changes_needed.push(SectionChangesNeeded::RenderWithPages); + // Nothing else we can do + return changes_needed; + } + + // Any other change will trigger a re-rendering of the section page only + changes_needed.push(SectionChangesNeeded::Render); + changes_needed +} + +/// Evaluates all the params in the front matter that changed so we can do the smallest +/// delta in the serve command +fn find_page_front_matter_changes(current: &PageFrontMatter, other: &PageFrontMatter) -> Vec { + let mut changes_needed = vec![]; + + if current.tags != other.tags { + changes_needed.push(PageChangesNeeded::Tags); + } + + if current.category != other.category { + changes_needed.push(PageChangesNeeded::Categories); + } + + if current.date != other.date || current.order != other.order { + changes_needed.push(PageChangesNeeded::Sort); + } + + changes_needed.push(PageChangesNeeded::Render); + changes_needed +} + +// What happens when a section or a page is changed +pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> { + let is_section = path.file_name().unwrap() == "_index.md"; + + // A page or section got deleted + if !path.exists() { + if is_section { + // A section was deleted, many things can be impacted: + // - the pages of the section are becoming orphans + // - any page that was referencing the section (index, etc) + let relative_path = site.sections[path].relative_path.clone(); + // Remove the link to it and the section itself from the Site + site.permalinks.remove(&relative_path); + site.sections.remove(path); + site.populate_sections(); + } else { + // A page was deleted, many things can be impacted: + // - the section the page is in + // - any page that was referencing the section (index, etc) + let relative_path = site.pages[path].relative_path.clone(); + site.permalinks.remove(&relative_path); + match site.pages.remove(path) { + Some(p) => { + if p.meta.has_tags() || p.meta.category.is_some() { + site.populate_tags_and_categories(); + } + + if site.find_parent_section(&p).is_some() { + site.populate_sections(); + } + }, + None => () + }; + } + // Deletion is something that doesn't happen all the time so we + // don't need to optimise it too much + return site.build(); + } + + // A section was edited + if is_section { + match site.add_section(path, true)? { + Some(prev) => { + // Updating a section + let current_meta = site.sections[path].meta.clone(); + // Front matter didn't change, only content did + // so we render only the section page, not its pages + if current_meta == prev.meta { + return site.render_section(&site.sections[path], false); + } + + // Front matter changed + for changes in find_section_front_matter_changes(¤t_meta, &prev.meta) { + // Sort always comes first if present so the rendering will be fine + match changes { + SectionChangesNeeded::Sort => site.sort_sections_pages(Some(path)), + SectionChangesNeeded::Render => site.render_section(&site.sections[path], false)?, + SectionChangesNeeded::RenderWithPages => site.render_section(&site.sections[path], true)?, + // can't be arsed to make the Delete efficient, it's not a common enough operation + SectionChangesNeeded::Delete => { + site.populate_sections(); + site.build()?; + }, + }; + } + return Ok(()); + }, + None => { + // New section, only render that one + site.populate_sections(); + return site.render_section(&site.sections[path], true); + } + }; + } + + // A page was edited + match site.add_page(path, true)? { + Some(prev) => { + // Updating a page + let current = site.pages[path].clone(); + // Front matter didn't change, only content did + // so we render only the section page, not its pages + if current.meta == prev.meta { + return site.render_page(&site.pages[path]); + } + + // Front matter changed + for changes in find_page_front_matter_changes(¤t.meta, &prev.meta) { + // Sort always comes first if present so the rendering will be fine + match changes { + PageChangesNeeded::Tags => { + site.populate_tags_and_categories(); + site.render_tags()?; + }, + PageChangesNeeded::Categories => { + site.populate_tags_and_categories(); + site.render_categories()?; + }, + PageChangesNeeded::Sort => { + let section_path = match site.find_parent_section(&site.pages[path]) { + Some(s) => s.file_path.clone(), + None => continue // Do nothing if it's an orphan page + }; + site.populate_sections(); + site.sort_sections_pages(Some(§ion_path)); + site.render_index()?; + }, + PageChangesNeeded::Render => { + site.render_page(&site.pages[path])?; + }, + }; + } + return Ok(()); + + }, + None => { + // It's a new page! + site.populate_sections(); + site.populate_tags_and_categories(); + // No need to optimise that yet, we can revisit if it becomes an issue + site.build()?; + } + } + + Ok(()) +} + +/// What happens when a template is changed +pub fn after_template_change(site: &mut Site, path: &Path) -> Result<()> { + site.tera.full_reload()?; + + match path.file_name().unwrap().to_str().unwrap() { + "sitemap.xml" => site.render_sitemap(), + "rss.xml" => site.render_rss_feed(), + "robots.txt" => site.render_robots(), + "categories.html" | "category.html" => site.render_categories(), + "tags.html" | "tag.html" => site.render_tags(), + "page.html" => { + site.render_sections()?; + site.render_orphan_pages() + }, + "section.html" => site.render_sections(), + // Either the index or some unknown template changed + // We can't really know what this change affects so rebuild all + // the things + _ => { + site.render_sections()?; + site.render_orphan_pages()?; + site.render_categories()?; + site.render_tags() + }, + } +} diff --git a/src/front_matter/page.rs b/src/front_matter/page.rs index 3596b421..07c1fb26 100644 --- a/src/front_matter/page.rs +++ b/src/front_matter/page.rs @@ -26,11 +26,11 @@ pub struct PageFrontMatter { pub tags: Option>, /// Whether this page is a draft and should be published or not pub draft: Option, - /// Only one category allowed + /// Only one category allowed. Can't be an empty string if present pub category: Option, /// Integer to use to order content. Lowest is at the bottom, highest first pub order: Option, - /// Optional template, if we want to specify which template to render for that page + /// Specify a template different from `page.html` to use for that page #[serde(skip_serializing)] pub template: Option, /// Any extra parameter present in the front matter @@ -56,6 +56,12 @@ impl PageFrontMatter { } } + if let Some(ref category) = f.category { + if category == "" { + bail!("`category` can't be empty if present") + } + } + Ok(f) } @@ -76,6 +82,13 @@ impl PageFrontMatter { pub fn order(&self) -> usize { self.order.unwrap() } + + pub fn has_tags(&self) -> bool { + match self.tags { + Some(ref t) => !t.is_empty(), + None => false + } + } } impl Default for PageFrontMatter { diff --git a/src/page.rs b/src/page.rs index 49384b35..cb43ff9e 100644 --- a/src/page.rs +++ b/src/page.rs @@ -297,14 +297,7 @@ pub fn sort_pages(pages: Vec, sort_by: SortBy) -> (Vec, Vec) { (can_be_sorted, cannot_be_sorted) }, - SortBy::None => { - let mut p = vec![]; - for page in pages { - p.push(page); - } - - (p, vec![]) - }, + SortBy::None => (pages, vec![]) } } diff --git a/src/section.rs b/src/section.rs index e8a408fb..7310ed8c 100644 --- a/src/section.rs +++ b/src/section.rs @@ -134,6 +134,23 @@ impl Section { paths.extend(self.ignored_pages.iter().map(|p| p.file_path.clone())); paths } + + /// Whether the page given belongs to that section + pub fn is_child_page(&self, page: &Page) -> bool { + for p in &self.pages { + if p.file_path == page.file_path { + return true; + } + } + + for p in &self.ignored_pages { + if p.file_path == page.file_path { + return true; + } + } + + false + } } impl ser::Serialize for Section { diff --git a/src/site.rs b/src/site.rs index d0d0e7c8..d57fe14a 100644 --- a/src/site.rs +++ b/src/site.rs @@ -18,8 +18,6 @@ use front_matter::{SortBy}; use templates::{GUTENBERG_TERA, global_fns, render_redirect_template}; - - #[derive(Debug, PartialEq)] enum RenderList { Tags, @@ -56,6 +54,8 @@ pub struct Site { static_path: PathBuf, pub tags: HashMap>, pub categories: HashMap>, + /// A map of all .md files (section and pages) and their permalink + /// We need that if there are relative links in the content that need to be resolved pub permalinks: HashMap, } @@ -92,6 +92,7 @@ impl Site { } /// Gets the path of all ignored pages in the site + /// Used for reporting them in the CLI pub fn get_ignored_pages(&self) -> Vec { self.sections .values() @@ -117,6 +118,17 @@ impl Site { orphans } + /// Finds the section that contains the page given if there is one + pub fn find_parent_section(&self, page: &Page) -> Option<&Section> { + for section in self.sections.values() { + if section.is_child_page(page) { + return Some(section) + } + } + + None + } + /// Used by tests to change the output path to a tmp dir #[doc(hidden)] pub fn set_output_path>(&mut self, path: P) { @@ -132,9 +144,9 @@ impl Site { for entry in glob(&content_glob).unwrap().filter_map(|e| e.ok()) { let path = entry.as_path(); if path.file_name().unwrap() == "_index.md" { - self.add_section(path)?; + self.add_section(path, false)?; } else { - self.add_page(path)?; + self.add_page(path, false)?; } } // Insert a default index section so we don't need to create a _index.md to render @@ -146,27 +158,15 @@ impl Site { self.sections.insert(index_path, index_section); } - // A map of all .md files (section and pages) and their permalink - // We need that if there are relative links in the content that need to be resolved - let mut permalinks = HashMap::new(); - - for page in self.pages.values() { - permalinks.insert(page.relative_path.clone(), page.permalink.clone()); - } - - for section in self.sections.values() { - permalinks.insert(section.relative_path.clone(), section.permalink.clone()); - } - + // TODO: make that parallel for page in self.pages.values_mut() { - page.render_markdown(&permalinks, &self.tera, &self.config)?; + page.render_markdown(&self.permalinks, &self.tera, &self.config)?; } - + // TODO: make that parallel for section in self.sections.values_mut() { - section.render_markdown(&permalinks, &self.tera, &self.config)?; + section.render_markdown(&self.permalinks, &self.tera, &self.config)?; } - self.permalinks = permalinks; self.populate_sections(); self.populate_tags_and_categories(); @@ -175,73 +175,82 @@ impl Site { Ok(()) } - /// Simple wrapper fn to avoid repeating that code in several places - fn add_page(&mut self, path: &Path) -> Result<()> { + /// Add a page to the site + /// The `render` parameter is used in the serve command, when rebuilding a page. + /// If `true`, it will also render the markdown for that page + /// Returns the previous page struct if there was one + pub fn add_page(&mut self, path: &Path, render: bool) -> Result> { let page = Page::from_file(&path, &self.config)?; - self.pages.insert(page.file_path.clone(), page); - Ok(()) - } - - /// Simple wrapper fn to avoid repeating that code in several places - fn add_section(&mut self, path: &Path) -> Result<()> { - let section = Section::from_file(path, &self.config)?; - self.sections.insert(section.file_path.clone(), section); - Ok(()) - } - - /// Called in serve, add the section and render it - fn add_section_and_render(&mut self, path: &Path) -> Result<()> { - self.add_section(path)?; - let mut section = self.sections.get_mut(path).unwrap(); - self.permalinks.insert(section.relative_path.clone(), section.permalink.clone()); - section.render_markdown(&self.permalinks, &self.tera, &self.config)?; - Ok(()) - } - - /// Called in serve, add a page again updating permalinks and its content - /// The bool in the result is whether the front matter has been updated or not - /// TODO: the above is very confusing, change that - fn add_page_and_render(&mut self, path: &Path) -> Result<(bool, Page)> { - let existing_page = self.pages.get(path).cloned(); - self.add_page(path)?; - let mut page = self.pages.get_mut(path).unwrap(); self.permalinks.insert(page.relative_path.clone(), page.permalink.clone()); - page.render_markdown(&self.permalinks, &self.tera, &self.config)?; + let prev = self.pages.insert(page.file_path.clone(), page); - if let Some(prev_page) = existing_page { - return Ok((prev_page.meta != page.meta, page.clone())); + if render { + let mut page = self.pages.get_mut(path).unwrap(); + page.render_markdown(&self.permalinks, &self.tera, &self.config)?; } - Ok((true, page.clone())) + + Ok(prev) + } + + /// Add a section to the site + /// The `render` parameter is used in the serve command, when rebuilding a page. + /// If `true`, it will also render the markdown for that page + /// Returns the previous page struct if there was one + pub fn add_section(&mut self, path: &Path, render: bool) -> Result> { + let section = Section::from_file(path, &self.config)?; + self.permalinks.insert(section.relative_path.clone(), section.permalink.clone()); + let prev = self.sections.insert(section.file_path.clone(), section); + + if render { + let mut section = self.sections.get_mut(path).unwrap(); + section.render_markdown(&self.permalinks, &self.tera, &self.config)?; + } + + Ok(prev) } /// Find out the direct subsections of each subsection if there are some /// as well as the pages for each section pub fn populate_sections(&mut self) { + let mut grandparent_paths = HashMap::new(); + for section in self.sections.values_mut() { + if let Some(grand_parent) = section.parent_path.parent() { + grandparent_paths.entry(grand_parent.to_path_buf()).or_insert_with(|| vec![]).push(section.clone()); + } + // Make sure the pages of a section are empty since we can call that many times on `serve` + section.pages = vec![]; + section.ignored_pages = vec![]; + } + for page in self.pages.values() { if self.sections.contains_key(&page.parent_path.join("_index.md")) { self.sections.get_mut(&page.parent_path.join("_index.md")).unwrap().pages.push(page.clone()); } } - let mut grandparent_paths = HashMap::new(); - for section in self.sections.values() { - if let Some(grand_parent) = section.parent_path.parent() { - grandparent_paths.entry(grand_parent.to_path_buf()).or_insert_with(|| vec![]).push(section.clone()); - } - } - for section in self.sections.values_mut() { - // TODO: avoid this clone - let (mut sorted_pages, cannot_be_sorted_pages) = sort_pages(section.pages.clone(), section.meta.sort_by()); - sorted_pages = populate_previous_and_next_pages(&sorted_pages); - section.pages = sorted_pages; - section.ignored_pages = cannot_be_sorted_pages; - match grandparent_paths.get(§ion.parent_path) { Some(paths) => section.subsections.extend(paths.clone()), None => continue, }; } + + self.sort_sections_pages(None); + } + + /// Sorts the pages of the section at the given path + /// By default will sort all sections but can be made to only sort a single one by providing a path + pub fn sort_sections_pages(&mut self, only: Option<&Path>) { + for (path, section) in self.sections.iter_mut() { + if let Some(p) = only { + if p != path { + continue; + } + } + let (sorted_pages, cannot_be_sorted_pages) = sort_pages(section.pages.clone(), section.meta.sort_by()); + section.pages = populate_previous_and_next_pages(&sorted_pages); + section.ignored_pages = cannot_be_sorted_pages; + } } /// Separated from `parse` for easier testing @@ -277,7 +286,7 @@ impl Site { html } - pub fn ensure_public_directory_exists(&self) -> Result<()> { + fn ensure_public_directory_exists(&self) -> Result<()> { let public = self.output_path.clone(); if !public.exists() { create_directory(&public)?; @@ -324,58 +333,6 @@ impl Site { Ok(()) } - pub fn rebuild_after_content_change(&mut self, path: &Path) -> Result<()> { - let is_section = path.ends_with("_index.md"); - - if path.exists() { - // file exists, either a new one or updating content - if is_section { - self.add_section_and_render(path)?; - self.render_sections()?; - } else { - // probably just an update so just re-parse that page - let (frontmatter_changed, page) = self.add_page_and_render(path)?; - // TODO: can probably be smarter and check what changed - if frontmatter_changed { - self.populate_sections(); - self.populate_tags_and_categories(); - self.build()?; - } else { - self.render_page(&page)?; - } - } - } else { - // File doesn't exist -> a deletion so we remove it from everything - let relative_path = if is_section { - self.sections[path].relative_path.clone() - } else { - self.pages[path].relative_path.clone() - }; - self.permalinks.remove(&relative_path); - - if is_section { - self.sections.remove(path); - } else { - self.pages.remove(path); - } - // TODO: probably no need to do that, we should be able to only re-render a page or a section. - self.populate_sections(); - self.populate_tags_and_categories(); - self.build()?; - } - - Ok(()) - } - - pub fn rebuild_after_template_change(&mut self, path: &Path) -> Result<()> { - self.tera.full_reload()?; - match path.file_name().unwrap().to_str().unwrap() { - "sitemap.xml" => self.render_sitemap(), - "rss.xml" => self.render_rss_feed(), - _ => self.build() // TODO: change that - } - } - /// Renders a single content page pub fn render_page(&self, page: &Page) -> Result<()> { self.ensure_public_directory_exists()?; @@ -417,18 +374,16 @@ impl Site { self.render_rss_feed()?; } self.render_robots()?; - if self.config.generate_categories_pages.unwrap() { - self.render_categories_and_tags(RenderList::Categories)?; - } - if self.config.generate_tags_pages.unwrap() { - self.render_categories_and_tags(RenderList::Tags)?; - } + // `render_categories` and `render_tags` will check whether the config allows + // them to render or not + self.render_categories()?; + self.render_tags()?; self.copy_static_directory() } /// Renders robots.txt - fn render_robots(&self) -> Result<()> { + pub fn render_robots(&self) -> Result<()> { self.ensure_public_directory_exists()?; create_file( self.output_path.join("robots.txt"), @@ -436,8 +391,27 @@ impl Site { ) } + /// Renders all categories if the config allows it + pub fn render_categories(&self) -> Result<()> { + if self.config.generate_categories_pages.unwrap() { + self.render_categories_and_tags(RenderList::Categories) + } else { + Ok(()) + } + } + + /// Renders all tags if the config allows it + pub fn render_tags(&self) -> Result<()> { + if self.config.generate_tags_pages.unwrap() { + self.render_categories_and_tags(RenderList::Tags) + } else { + Ok(()) + } + } + /// Render the /{categories, list} pages and each individual category/tag page /// They are the same thing fundamentally, a list of pages with something in common + /// TODO: revisit this function, lots of things have changed since then fn render_categories_and_tags(&self, kind: RenderList) -> Result<()> { let items = match kind { RenderList::Categories => &self.categories, @@ -509,7 +483,8 @@ impl Site { Ok(()) } - fn render_sitemap(&self) -> Result<()> { + /// What it says on the tin + pub fn render_sitemap(&self) -> Result<()> { self.ensure_public_directory_exists()?; let mut context = Context::new(); context.add("pages", &self.pages.values().collect::>()); @@ -544,7 +519,7 @@ impl Site { Ok(()) } - fn render_rss_feed(&self) -> Result<()> { + pub fn render_rss_feed(&self) -> Result<()> { self.ensure_public_directory_exists()?; let mut context = Context::new(); @@ -587,7 +562,7 @@ impl Site { } /// Renders a single section - fn render_section(&self, section: &Section) -> Result<()> { + pub fn render_section(&self, section: &Section, render_pages: bool) -> Result<()> { self.ensure_public_directory_exists()?; let public = self.output_path.clone(); @@ -600,8 +575,10 @@ impl Site { } } - for page in §ion.pages { - self.render_page(page)?; + if render_pages { + for page in §ion.pages { + self.render_page(page)?; + } } if !section.meta.should_render() { @@ -622,16 +599,20 @@ impl Site { Ok(()) } + pub fn render_index(&self) -> Result<()> { + self.render_section(&self.sections[&self.base_path.join("content").join("_index.md")], false) + } + /// Renders all sections - fn render_sections(&self) -> Result<()> { + pub fn render_sections(&self) -> Result<()> { for section in self.sections.values() { - self.render_section(section)?; + self.render_section(section, true)?; } Ok(()) } /// Renders all pages that do not belong to any sections - fn render_orphan_pages(&self) -> Result<()> { + pub fn render_orphan_pages(&self) -> Result<()> { self.ensure_public_directory_exists()?; for page in self.get_all_orphan_pages() {