emelie/zola
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

imageproc things for paths refactor (#1484)

Browse source 
* Move image loading code used by get_image_metadata from templates to imageproc

* resize_image: Compute image resize op beforehand and return metadata to caller

* get_image_metdata: Use a serializable response struct

* imageproc: Add integration tests

* imageproc: Assume webp is lossy

* get_image_metadata: Use webp crate for decoding WebP, image is still not very good at it

* imageproc: Test read_image_dimensions in integration tests

* imageproc: Rename read_image_dimensions() as read_image_metadata()

* imageproc: Fix a regression in hash computation

* imageproc: Don't hardcode hashes in integration tests, pattern match them instead

* imageproc: Style fixes

* imageproc: Fix integration tests on Windows (hopefully)
This commit is contained in:
Vojtech Kral 2021-06-08 09:38:29 +02:00 committed by Vincent Prouillet
parent 009d105210
commit 9145af6b3e
10 changed files with 520 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Cargo.lock generated
View file

@ -1028,6 +1028,10 @@ dependencies = [
"lazy_static", "lazy_static",
"rayon", "rayon",
"regex", "regex",
"serde",
"serde_json",
"site",
"svg_metadata",
"tera", "tera",
"utils", "utils",
"webp", "webp",
@ -2585,7 +2589,6 @@ dependencies = [
"config", "config",
"csv", "csv",
"errors", "errors",
"image",
"imageproc", "imageproc",
"lazy_static", "lazy_static",
"library", "library",
@ -2597,7 +2600,6 @@ dependencies = [
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"sha2", "sha2",
"svg_metadata",
"tempfile", "tempfile",
"tera", "tera",
"toml", "toml",

7
components/imageproc/Cargo.toml
View file

@ -11,7 +11,14 @@ tera = "1"
image = "0.23" image = "0.23"
rayon = "1" rayon = "1"
webp = "0.1.1" webp = "0.1.1"
serde = { version = "1", features = ["derive"] }
svg_metadata = "0.4.1"
errors = { path = "../errors" } errors = { path = "../errors" }
utils = { path = "../utils" } utils = { path = "../utils" }
config = { path = "../config" } config = { path = "../config" }
[dev-dependencies]
# TODO: prune
serde_json = "1"
site = { path = "../site" }

390
components/imageproc/src/lib.rs
View file

@ -1,15 +1,21 @@
use std::collections::hash_map::Entry as HEntry; use std::collections::hash_map::Entry as HEntry;
use std::collections::HashMap; use std::collections::HashMap;
use std::error::Error as StdError;
use std::ffi::OsStr;
use std::fs::{self, File}; use std::fs::{self, File};
use std::hash::{Hash, Hasher}; use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::{collections::hash_map::DefaultHasher, io::Write}; use std::{collections::hash_map::DefaultHasher, io::Write};
use image::error::ImageResult;
use image::io::Reader as ImgReader;
use image::{imageops::FilterType, EncodableLayout}; use image::{imageops::FilterType, EncodableLayout};
use image::{GenericImageView, ImageOutputFormat}; use image::{ImageFormat, ImageOutputFormat};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use rayon::prelude::*; use rayon::prelude::*;
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize};
use svg_metadata::Metadata as SvgMetadata;
use config::Config; use config::Config;
use errors::{Error, Result}; use errors::{Error, Result};
@ -23,9 +29,34 @@ lazy_static! {
Regex::new(r#"([0-9a-f]{16})([0-9a-f]{2})[.](jpg|png|webp)"#).unwrap(); Regex::new(r#"([0-9a-f]{16})([0-9a-f]{2})[.](jpg|png|webp)"#).unwrap();
} }
/// Describes the precise kind of a resize operation /// Size and format read cheaply with `image`'s `Reader`.
#[derive(Debug)]
struct ImageMeta {
size: (u32, u32),
format: Option<ImageFormat>,
}
impl ImageMeta {
fn read(path: &Path) -> ImageResult<Self> {
let reader = ImgReader::open(path).and_then(ImgReader::with_guessed_format)?;
let format = reader.format();
let size = reader.into_dimensions()?;
Ok(Self { size, format })
}
fn is_lossy(&self) -> bool {
use ImageFormat::*;
// We assume lossy by default / if unknown format
let format = self.format.unwrap_or(Jpeg);
!matches!(format, Png | Pnm | Tiff | Tga | Bmp | Ico | Hdr | Farbfeld)
}
}
/// De-serialized & sanitized arguments of `resize_image`
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResizeOp { pub enum ResizeArgs {
/// A simple scale operation that doesn't take aspect ratio into account /// A simple scale operation that doesn't take aspect ratio into account
Scale(u32, u32), Scale(u32, u32),
/// Scales the image to a specified width with height computed such /// Scales the image to a specified width with height computed such
@ -45,9 +76,9 @@ pub enum ResizeOp {
Fill(u32, u32), Fill(u32, u32),
} }
impl ResizeOp { impl ResizeArgs {
pub fn from_args(op: &str, width: Option<u32>, height: Option<u32>) -> Result<ResizeOp> { pub fn from_args(op: &str, width: Option<u32>, height: Option<u32>) -> Result<Self> {
use ResizeOp::*; use ResizeArgs::*;
// Validate args: // Validate args:
match op { match op {
@ -80,58 +111,87 @@ impl ResizeOp {
_ => unreachable!(), _ => unreachable!(),
}) })
} }
}
pub fn width(self) -> Option<u32> { /// Contains image crop/resize instructions for use by `Processor`
use ResizeOp::*; ///
/// The `Processor` applies `crop` first, if any, and then `resize`, if any.
#[derive(Clone, PartialEq, Eq, Hash, Default, Debug)]
struct ResizeOp {
crop: Option<(u32, u32, u32, u32)>, // x, y, w, h
resize: Option<(u32, u32)>, // w, h
}
match self { impl ResizeOp {
Scale(w, _) => Some(w), fn new(args: ResizeArgs, (orig_w, orig_h): (u32, u32)) -> Self {
FitWidth(w) => Some(w), use ResizeArgs::*;
FitHeight(_) => None,
Fit(w, _) => Some(w), let res = ResizeOp::default();
Fill(w, _) => Some(w),
match args {
Scale(w, h) => res.resize((w, h)),
FitWidth(w) => {
let h = (orig_h as u64 * w as u64) / orig_w as u64;
res.resize((w, h as u32))
}
FitHeight(h) => {
let w = (orig_w as u64 * h as u64) / orig_h as u64;
res.resize((w as u32, h))
}
Fit(w, h) => {
let orig_w_h = orig_w as u64 * h as u64;
let orig_h_w = orig_h as u64 * w as u64;
if orig_w_h > orig_h_w {
Self::new(FitWidth(w), (orig_w, orig_h))
} else {
Self::new(FitHeight(h), (orig_w, orig_h))
}
}
Fill(w, h) => {
const RATIO_EPSILLION: f32 = 0.1;
let factor_w = orig_w as f32 / w as f32;
let factor_h = orig_h as f32 / h as f32;
if (factor_w - factor_h).abs() <= RATIO_EPSILLION {
// If the horizontal and vertical factor is very similar,
// that means the aspect is similar enough that there's not much point
// in cropping, so just perform a simple scale in this case.
res.resize((w, h))
} else {
// We perform the fill such that a crop is performed first
// and then resize_exact can be used, which should be cheaper than
// resizing and then cropping (smaller number of pixels to resize).
let (crop_w, crop_h) = if factor_w < factor_h {
(orig_w, (factor_w * h as f32).round() as u32)
} else {
((factor_h * w as f32).round() as u32, orig_h)
};
let (offset_w, offset_h) = if factor_w < factor_h {
(0, (orig_h - crop_h) / 2)
} else {
((orig_w - crop_w) / 2, 0)
};
res.crop((offset_w, offset_h, crop_w, crop_h)).resize((w, h))
}
}
} }
} }
pub fn height(self) -> Option<u32> { fn crop(mut self, crop: (u32, u32, u32, u32)) -> Self {
use ResizeOp::*; self.crop = Some(crop);
self
}
match self { fn resize(mut self, size: (u32, u32)) -> Self {
Scale(_, h) => Some(h), self.resize = Some(size);
FitWidth(_) => None, self
FitHeight(h) => Some(h),
Fit(_, h) => Some(h),
Fill(_, h) => Some(h),
}
} }
} }
impl From<ResizeOp> for u8 {
fn from(op: ResizeOp) -> u8 {
use ResizeOp::*;
match op {
Scale(_, _) => 1,
FitWidth(_) => 2,
FitHeight(_) => 3,
Fit(_, _) => 4,
Fill(_, _) => 5,
}
}
}
#[allow(clippy::derive_hash_xor_eq)]
impl Hash for ResizeOp {
fn hash<H: Hasher>(&self, hasher: &mut H) {
hasher.write_u8(u8::from(*self));
if let Some(w) = self.width() {
hasher.write_u32(w);
}
if let Some(h) = self.height() {
hasher.write_u32(h);
}
}
}
/// Thumbnail image format /// Thumbnail image format
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Format { pub enum Format {
@ -144,21 +204,23 @@ pub enum Format {
} }
impl Format { impl Format {
pub fn from_args(source: &str, format: &str, quality: Option<u8>) -> Result<Format> { fn from_args(meta: &ImageMeta, format: &str, quality: Option<u8>) -> Result<Format> {
use Format::*; use Format::*;
if let Some(quality) = quality { if let Some(quality) = quality {
assert!(quality > 0 && quality <= 100, "Quality must be within the range [1; 100]"); assert!(quality > 0 && quality <= 100, "Quality must be within the range [1; 100]");
} }
let jpg_quality = quality.unwrap_or(DEFAULT_Q_JPG); let jpg_quality = quality.unwrap_or(DEFAULT_Q_JPG);
match format { match format {
"auto" => match Self::is_lossy(source) { "auto" => {
Some(true) => Ok(Jpeg(jpg_quality)), if meta.is_lossy() {
Some(false) => Ok(Png), Ok(Jpeg(jpg_quality))
None => Err(format!("Unsupported image file: {}", source).into()), } else {
}, Ok(Png)
}
}
"jpeg" | "jpg" => Ok(Jpeg(jpg_quality)), "jpeg" | "jpg" => Ok(Jpeg(jpg_quality)),
"png" => Ok(Png), "png" => Ok(Png),
"webp" => Ok(WebP(quality)), "webp" => Ok(WebP(quality)), // FIXME: this is undoc'd
_ => Err(format!("Invalid image format: {}", format).into()), _ => Err(format!("Invalid image format: {}", format).into()),
} }
} }
@ -173,8 +235,8 @@ impl Format {
"png" => Some(false), "png" => Some(false),
"gif" => Some(false), "gif" => Some(false),
"bmp" => Some(false), "bmp" => Some(false),
// It is assumed that webp is lossless, but it can be both // It is assumed that webp is lossy, but it can be both
"webp" => Some(false), "webp" => Some(true),
_ => None, _ => None,
}) })
.unwrap_or(None) .unwrap_or(None)
@ -212,7 +274,10 @@ impl Hash for Format {
/// Holds all data needed to perform a resize operation /// Holds all data needed to perform a resize operation
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct ImageOp { pub struct ImageOp {
source: String, /// This is the source input path string as passed in the template, we need this to compute the hash.
/// Hashing the resolved `input_path` would include the absolute path to the image
/// with all filesystem components.
input_src: String,
input_path: PathBuf, input_path: PathBuf,
op: ResizeOp, op: ResizeOp,
format: Format, format: Format,
@ -226,79 +291,32 @@ pub struct ImageOp {
} }
impl ImageOp { impl ImageOp {
pub fn from_args( const RESIZE_FILTER: FilterType = FilterType::Lanczos3;
source: String,
input_path: PathBuf,
op: &str,
width: Option<u32>,
height: Option<u32>,
format: &str,
quality: Option<u8>,
) -> Result<ImageOp> {
let op = ResizeOp::from_args(op, width, height)?;
let format = Format::from_args(&source, format, quality)?;
fn new(input_src: String, input_path: PathBuf, op: ResizeOp, format: Format) -> ImageOp {
let mut hasher = DefaultHasher::new(); let mut hasher = DefaultHasher::new();
hasher.write(source.as_ref()); hasher.write(input_src.as_ref());
op.hash(&mut hasher); op.hash(&mut hasher);
format.hash(&mut hasher); format.hash(&mut hasher);
let hash = hasher.finish(); let hash = hasher.finish();
Ok(ImageOp { source, input_path, op, format, hash, collision_id: 0 }) ImageOp { input_src, input_path, op, format, hash, collision_id: 0 }
} }
fn perform(&self, target_path: &Path) -> Result<()> { fn perform(&self, target_path: &Path) -> Result<()> {
use ResizeOp::*;
if !ufs::file_stale(&self.input_path, target_path) { if !ufs::file_stale(&self.input_path, target_path) {
return Ok(()); return Ok(());
} }
let mut img = image::open(&self.input_path)?; let mut img = image::open(&self.input_path)?;
let (img_w, img_h) = img.dimensions();
const RESIZE_FILTER: FilterType = FilterType::Lanczos3; let img = match self.op.crop {
const RATIO_EPSILLION: f32 = 0.1; Some((x, y, w, h)) => img.crop(x, y, w, h),
None => img,
let img = match self.op { };
Scale(w, h) => img.resize_exact(w, h, RESIZE_FILTER), let img = match self.op.resize {
FitWidth(w) => img.resize(w, u32::MAX, RESIZE_FILTER), Some((w, h)) => img.resize_exact(w, h, Self::RESIZE_FILTER),
FitHeight(h) => img.resize(u32::MAX, h, RESIZE_FILTER), None => img,
Fit(w, h) => {
if img_w > w || img_h > h {
img.resize(w, h, RESIZE_FILTER)
} else {
img
}
}
Fill(w, h) => {
let factor_w = img_w as f32 / w as f32;
let factor_h = img_h as f32 / h as f32;
if (factor_w - factor_h).abs() <= RATIO_EPSILLION {
// If the horizontal and vertical factor is very similar,
// that means the aspect is similar enough that there's not much point
// in cropping, so just perform a simple scale in this case.
img.resize_exact(w, h, RESIZE_FILTER)
} else {
// We perform the fill such that a crop is performed first
// and then resize_exact can be used, which should be cheaper than
// resizing and then cropping (smaller number of pixels to resize).
let (crop_w, crop_h) = if factor_w < factor_h {
(img_w, (factor_w * h as f32).round() as u32)
} else {
((factor_h * w as f32).round() as u32, img_h)
};
let (offset_w, offset_h) = if factor_w < factor_h {
(0, (img_h - crop_h) / 2)
} else {
((img_w - crop_w) / 2, 0)
};
img.crop(offset_w, offset_h, crop_w, crop_h).resize_exact(w, h, RESIZE_FILTER)
}
}
}; };
let mut f = File::create(target_path)?; let mut f = File::create(target_path)?;
@ -324,6 +342,33 @@ impl ImageOp {
} }
} }
// FIXME: Explain this in the doc
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct EnqueueResponse {
/// The final URL for that asset
pub url: String,
/// The path to the static asset generated
pub static_path: String,
/// New image width
pub width: u32,
/// New image height
pub height: u32,
/// Original image width
pub orig_width: u32,
/// Original image height
pub orig_height: u32,
}
impl EnqueueResponse {
fn new(url: String, static_path: PathBuf, meta: &ImageMeta, op: &ResizeOp) -> Self {
let static_path = static_path.to_string_lossy().into_owned();
let (width, height) = op.resize.unwrap_or(meta.size);
let (orig_width, orig_height) = meta.size;
Self { url, static_path, width, height, orig_width, orig_height }
}
}
/// A struct into which image operations can be enqueued and then performed. /// A struct into which image operations can be enqueued and then performed.
/// All output is written in a subdirectory in `static_path`, /// All output is written in a subdirectory in `static_path`,
/// taking care of file stale status based on timestamps and possible hash collisions. /// taking care of file stale status based on timestamps and possible hash collisions.
@ -360,6 +405,29 @@ impl Processor {
self.img_ops.len() + self.img_ops_collisions.len() self.img_ops.len() + self.img_ops_collisions.len()
} }
pub fn enqueue(
&mut self,
input_src: String,
input_path: PathBuf,
op: &str,
width: Option<u32>,
height: Option<u32>,
format: &str,
quality: Option<u8>,
) -> Result<EnqueueResponse> {
let meta = ImageMeta::read(&input_path).map_err(|e| {
Error::chain(format!("Failed to read image: {}", input_path.display()), e)
})?;
let args = ResizeArgs::from_args(op, width, height)?;
let op = ResizeOp::new(args, meta.size);
let format = Format::from_args(&meta, format, quality)?;
let img_op = ImageOp::new(input_src, input_path, op.clone(), format);
let (static_path, url) = self.insert(img_op);
Ok(EnqueueResponse::new(url, static_path, &meta, &op))
}
fn insert_with_collisions(&mut self, mut img_op: ImageOp) -> u32 { fn insert_with_collisions(&mut self, mut img_op: ImageOp) -> u32 {
match self.img_ops.entry(img_op.hash) { match self.img_ops.entry(img_op.hash) {
HEntry::Occupied(entry) => { HEntry::Occupied(entry) => {
@ -414,7 +482,7 @@ impl Processor {
/// Adds the given operation to the queue but do not process it immediately. /// Adds the given operation to the queue but do not process it immediately.
/// Returns (path in static folder, final URL). /// Returns (path in static folder, final URL).
pub fn insert(&mut self, img_op: ImageOp) -> (PathBuf, String) { fn insert(&mut self, img_op: ImageOp) -> (PathBuf, String) {
let hash = img_op.hash; let hash = img_op.hash;
let format = img_op.format; let format = img_op.format;
let collision_id = self.insert_with_collisions(img_op); let collision_id = self.insert_with_collisions(img_op);
@ -423,6 +491,7 @@ impl Processor {
(Path::new("static").join(RESIZED_SUBDIR).join(filename), url) (Path::new("static").join(RESIZED_SUBDIR).join(filename), url)
} }
/// Remove stale processed images in the output directory
pub fn prune(&self) -> Result<()> { pub fn prune(&self) -> Result<()> {
// Do not create folders if they don't exist // Do not create folders if they don't exist
if !self.output_dir.exists() { if !self.output_dir.exists() {
@ -449,6 +518,7 @@ impl Processor {
Ok(()) Ok(())
} }
/// Run the enqueued image operations
pub fn do_process(&mut self) -> Result<()> { pub fn do_process(&mut self) -> Result<()> {
if !self.img_ops.is_empty() { if !self.img_ops.is_empty() {
ufs::ensure_directory_exists(&self.output_dir)?; ufs::ensure_directory_exists(&self.output_dir)?;
@ -459,9 +529,89 @@ impl Processor {
.map(|(hash, op)| { .map(|(hash, op)| {
let target = let target =
self.output_dir.join(Self::op_filename(*hash, op.collision_id, op.format)); self.output_dir.join(Self::op_filename(*hash, op.collision_id, op.format));
op.perform(&target) op.perform(&target).map_err(|e| {
.map_err(|e| Error::chain(format!("Failed to process image: {}", op.source), e)) Error::chain(format!("Failed to process image: {}", op.input_path.display()), e)
})
}) })
.collect::<Result<()>>() .collect::<Result<()>>()
} }
} }
#[derive(Debug, Serialize, Eq, PartialEq)]
pub struct ImageMetaResponse {
pub width: u32,
pub height: u32,
pub format: Option<&'static str>,
}
impl ImageMetaResponse {
pub fn new_svg(width: u32, height: u32) -> Self {
Self { width, height, format: Some("svg") }
}
}
impl From<ImageMeta> for ImageMetaResponse {
fn from(im: ImageMeta) -> Self {
Self {
width: im.size.0,
height: im.size.1,
format: im.format.and_then(|f| f.extensions_str().get(0)).map(|&f| f),
}
}
}
impl From<webp::WebPImage> for ImageMetaResponse {
fn from(img: webp::WebPImage) -> Self {
Self { width: img.width(), height: img.height(), format: Some("webp") }
}
}
/// Read image dimensions (cheaply), used in `get_image_metadata()`, supports SVG
pub fn read_image_metadata<P: AsRef<Path>>(path: P) -> Result<ImageMetaResponse> {
let path = path.as_ref();
let ext = path.extension().and_then(OsStr::to_str).unwrap_or("").to_lowercase();
let error = |e: Box<dyn StdError + Send + Sync>| {
Error::chain(format!("Failed to read image: {}", path.display()), e)
};
match ext.as_str() {
"svg" => {
let img = SvgMetadata::parse_file(&path).map_err(|e| error(e.into()))?;
match (img.height(), img.width(), img.view_box()) {
(Some(h), Some(w), _) => Ok((h, w)),
(_, _, Some(view_box)) => Ok((view_box.height, view_box.width)),
_ => Err("Invalid dimensions: SVG width/height and viewbox not set.".into()),
}
.map(|(h, w)| ImageMetaResponse::new_svg(h as u32, w as u32))
}
"webp" => {
// Unfortunatelly we have to load the entire image here, unlike with the others :|
let data = fs::read(path).map_err(|e| error(e.into()))?;
let decoder = webp::Decoder::new(&data[..]);
decoder
.decode()
.map(ImageMetaResponse::from)
.ok_or_else(|| Error::msg(format!("Failed to decode WebP image: {}", path.display())))
}
_ => ImageMeta::read(path).map(ImageMetaResponse::from).map_err(|e| error(e.into())),
}
}
/// Assert that `address` matches `prefix` + RESIZED_FILENAME regex + "." + `extension`,
/// this is useful in test so that we don't need to hardcode hash, which is annoying.
pub fn assert_processed_path_matches(path: &str, prefix: &str, extension: &str) {
let filename = path
.strip_prefix(prefix)
.unwrap_or_else(|| panic!("Path `{}` doesn't start with `{}`", path, prefix));
let suffix = format!(".{}", extension);
assert!(filename.ends_with(&suffix), "Path `{}` doesn't end with `{}`", path, suffix);
assert!(
RESIZED_FILENAME.is_match_at(filename, 0),
"In path `{}`, file stem `{}` doesn't match the RESIZED_FILENAME regex",
path,
filename
);
}

161
components/imageproc/tests/resize_image.rs Normal file
View file

@ -0,0 +1,161 @@
use std::env;
use std::path::{PathBuf, MAIN_SEPARATOR as SLASH};
use lazy_static::lazy_static;
use config::Config;
use imageproc::{assert_processed_path_matches, ImageMetaResponse, Processor};
use utils::fs as ufs;
static CONFIG: &str = r#"
title = "imageproc integration tests"
base_url = "https://example.com"
compile_sass = false
build_search_index = false
[markdown]
highlight_code = false
"#;
lazy_static! {
static ref TEST_IMGS: PathBuf =
[env!("CARGO_MANIFEST_DIR"), "tests", "test_imgs"].iter().collect();
static ref TMPDIR: PathBuf = {
let tmpdir = option_env!("CARGO_TARGET_TMPDIR").map(PathBuf::from).unwrap_or_else(|| {
env::current_exe().unwrap().parent().unwrap().parent().unwrap().join("tmpdir")
});
ufs::ensure_directory_exists(&tmpdir).unwrap();
tmpdir
};
static ref PROCESSED_PREFIX: String = format!("static{0}processed_images{0}", SLASH);
}
fn image_op_test(
source_img: &str,
op: &str,
width: Option<u32>,
height: Option<u32>,
format: &str,
expect_ext: &str,
expect_width: u32,
expect_height: u32,
orig_width: u32,
orig_height: u32,
) {
let source_path = TEST_IMGS.join(source_img);
let config = Config::parse(&CONFIG).unwrap();
let mut proc = Processor::new(TMPDIR.clone(), &config);
let resp =
proc.enqueue(source_img.into(), source_path, op, width, height, format, None).unwrap();
assert_processed_path_matches(&resp.url, "https://example.com/processed_images/", expect_ext);
assert_processed_path_matches(&resp.static_path, PROCESSED_PREFIX.as_str(), expect_ext);
assert_eq!(resp.width, expect_width);
assert_eq!(resp.height, expect_height);
assert_eq!(resp.orig_width, orig_width);
assert_eq!(resp.orig_height, orig_height);
proc.do_process().unwrap();
let processed_path = PathBuf::from(&resp.static_path);
let processed_size = imageproc::read_image_metadata(&TMPDIR.join(processed_path))
.map(|meta| (meta.width, meta.height))
.unwrap();
assert_eq!(processed_size, (expect_width, expect_height));
}
fn image_meta_test(source_img: &str) -> ImageMetaResponse {
let source_path = TEST_IMGS.join(source_img);
imageproc::read_image_metadata(&source_path).unwrap()
}
#[test]
fn resize_image_scale() {
image_op_test("jpg.jpg", "scale", Some(150), Some(150), "auto", "jpg", 150, 150, 300, 380);
}
#[test]
fn resize_image_fit_width() {
image_op_test("jpg.jpg", "fit_width", Some(150), None, "auto", "jpg", 150, 190, 300, 380);
}
#[test]
fn resize_image_fit_height() {
image_op_test("webp.webp", "fit_height", None, Some(190), "auto", "jpg", 150, 190, 300, 380);
}
#[test]
fn resize_image_fit1() {
image_op_test("jpg.jpg", "fit", Some(150), Some(200), "auto", "jpg", 150, 190, 300, 380);
}
#[test]
fn resize_image_fit2() {
image_op_test("jpg.jpg", "fit", Some(160), Some(180), "auto", "jpg", 142, 180, 300, 380);
}
#[test]
fn resize_image_fill1() {
image_op_test("jpg.jpg", "fill", Some(100), Some(200), "auto", "jpg", 100, 200, 300, 380);
}
#[test]
fn resize_image_fill2() {
image_op_test("jpg.jpg", "fill", Some(200), Some(100), "auto", "jpg", 200, 100, 300, 380);
}
#[test]
fn resize_image_png_png() {
image_op_test("png.png", "scale", Some(150), Some(150), "auto", "png", 150, 150, 300, 380);
}
#[test]
fn resize_image_png_jpg() {
image_op_test("png.png", "scale", Some(150), Some(150), "jpg", "jpg", 150, 150, 300, 380);
}
#[test]
fn resize_image_png_webp() {
image_op_test("png.png", "scale", Some(150), Some(150), "webp", "webp", 150, 150, 300, 380);
}
#[test]
fn resize_image_webp_jpg() {
image_op_test("webp.webp", "scale", Some(150), Some(150), "auto", "jpg", 150, 150, 300, 380);
}
#[test]
fn read_image_metadata_jpg() {
assert_eq!(
image_meta_test("jpg.jpg"),
ImageMetaResponse { width: 300, height: 380, format: Some("jpg") }
);
}
#[test]
fn read_image_metadata_png() {
assert_eq!(
image_meta_test("png.png"),
ImageMetaResponse { width: 300, height: 380, format: Some("png") }
);
}
#[test]
fn read_image_metadata_svg() {
assert_eq!(
image_meta_test("svg.svg"),
ImageMetaResponse { width: 300, height: 300, format: Some("svg") }
);
}
#[test]
fn read_image_metadata_webp() {
assert_eq!(
image_meta_test("webp.webp"),
ImageMetaResponse { width: 300, height: 380, format: Some("webp") }
);
}
// TODO: Test that hash remains the same if physical path is changed

BIN
components/imageproc/tests/test_imgs/jpg.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 47 KiB

BIN
components/imageproc/tests/test_imgs/png.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 120 KiB

56
components/imageproc/tests/test_imgs/svg.svg Normal file
View file

@ -0,0 +1,56 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="100%" height="100%" viewBox="0 0 300 300">
<title>SVG Logo</title>
<desc>Designed for the SVG Logo Contest in 2006 by Harvey Rayner, and adopted by W3C in 2009. It is available under the Creative Commons license for those who have an SVG product or who are using SVG on their site.</desc>
<metadata id="license">
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://web.resource.org/cc/">
<cc:Work rdf:about="">
<dc:title>SVG Logo</dc:title>
<dc:date>14-08-2009</dc:date>
<dc:creator>
<cc:Agent><dc:title>W3C</dc:title></cc:Agent>
<cc:Agent><dc:title>Harvey Rayner, designer</dc:title></cc:Agent>
</dc:creator>
<dc:description>See document description</dc:description>
<cc:license rdf:resource="http://creativecommons.org/licenses/by-nc-sa/2.5/"/>
<dc:format>image/svg+xml</dc:format>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
</cc:Work>
<cc:License rdf:about="http://creativecommons.org/licenses/by-nc-sa/2.5/">
<cc:permits rdf:resource="http://web.resource.org/cc/Reproduction"/>
<cc:permits rdf:resource="http://web.resource.org/cc/Distribution"/>
<cc:requires rdf:resource="http://web.resource.org/cc/Notice"/>
<cc:requires rdf:resource="http://web.resource.org/cc/Attribution"/>
<cc:prohibits rdf:resource="http://web.resource.org/cc/CommercialUse"/>
<cc:permits rdf:resource="http://web.resource.org/cc/DerivativeWorks"/>
<cc:requires rdf:resource="http://web.resource.org/cc/ShareAlike"/>
</cc:License>
</rdf:RDF>
</metadata>
<defs>
<g id="SVG" fill="#ffffff" transform="scale(2) translate(20,79)">
<path id="S" d="M 5.482,31.319 C2.163,28.001 0.109,23.419 0.109,18.358 C0.109,8.232 8.322,0.024 18.443,0.024 C28.569,0.024 36.782,8.232 36.782,18.358 L26.042,18.358 C26.042,14.164 22.638,10.765 18.443,10.765 C14.249,10.765 10.850,14.164 10.850,18.358 C10.850,20.453 11.701,22.351 13.070,23.721 L13.075,23.721 C14.450,25.101 15.595,25.500 18.443,25.952 L18.443,25.952 C23.509,26.479 28.091,28.006 31.409,31.324 L31.409,31.324 C34.728,34.643 36.782,39.225 36.782,44.286 C36.782,54.412 28.569,62.625 18.443,62.625 C8.322,62.625 0.109,54.412 0.109,44.286 L10.850,44.286 C10.850,48.480 14.249,51.884 18.443,51.884 C22.638,51.884 26.042,48.480 26.042,44.286 C26.042,42.191 25.191,40.298 23.821,38.923 L23.816,38.923 C22.441,37.548 20.468,37.074 18.443,36.697 L18.443,36.692 C13.533,35.939 8.800,34.638 5.482,31.319 L5.482,31.319 L5.482,31.319 Z"/>
<path id="V" d="M 73.452,0.024 L60.482,62.625 L49.742,62.625 L36.782,0.024 L47.522,0.024 L55.122,36.687 L62.712,0.024 L73.452,0.024 Z"/>
<path id="G" d="M 91.792,25.952 L110.126,25.952 L110.126,44.286 L110.131,44.286 C110.131,54.413 101.918,62.626 91.792,62.626 C81.665,62.626 73.458,54.413 73.458,44.286 L73.458,44.286 L73.458,18.359 L73.453,18.359 C73.453,8.233 81.665,0.025 91.792,0.025 C101.913,0.025 110.126,8.233 110.126,18.359 L99.385,18.359 C99.385,14.169 95.981,10.765 91.792,10.765 C87.597,10.765 84.198,14.169 84.198,18.359 L84.198,44.286 L84.198,44.286 C84.198,48.481 87.597,51.880 91.792,51.880 C95.981,51.880 99.380,48.481 99.385,44.291 L99.385,44.286 L99.385,36.698 L91.792,36.698 L91.792,25.952 L91.792,25.952 Z"/>
</g>
</defs>
<path id="base" fill="#000" d="M8.5,150 H291.5 V250 C291.5,273.5 273.5,291.5 250,291.5 H50 C26.5,291.5 8.5,273.5 8.5,250 Z"/>
<g stroke-width="38.0086" stroke="#000">
<g id="svgstar" transform="translate(150, 150)">
<path id="svgbar" fill="#ffb13b" d="M-84.1487,-15.8513 a22.4171,22.4171 0 1 0 0,31.7026 h168.2974 a22.4171,22.4171 0 1 0 0,-31.7026 Z"/>
<use xlink:href="#svgbar" transform="rotate(45)"/>
<use xlink:href="#svgbar" transform="rotate(90)"/>
<use xlink:href="#svgbar" transform="rotate(135)"/>
</g>
</g>
<use xlink:href="#svgstar"/>
<use xlink:href="#base" opacity="0.85"/>
<use xlink:href="#SVG"/>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 4.1 KiB

BIN
components/imageproc/tests/test_imgs/webp.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 9.6 KiB

2
components/templates/Cargo.toml
View file

@ -10,14 +10,12 @@ base64 = "0.13"
lazy_static = "1" lazy_static = "1"
toml = "0.5" toml = "0.5"
csv = "1" csv = "1"
image = "0.23"
serde = "1" serde = "1"
serde_json = "1" serde_json = "1"
serde_derive = "1" serde_derive = "1"
sha2 = "0.9" sha2 = "0.9"
url = "2" url = "2"
nom-bibtex = "0.3" nom-bibtex = "0.3"
svg_metadata = "0.4.1"
errors = { path = "../errors" } errors = { path = "../errors" }
utils = { path = "../utils" } utils = { path = "../utils" }

77
components/templates/src/global_fns/images.rs
View file

@ -1,23 +1,11 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::ffi::OsStr; use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use image::GenericImageView; use tera::{from_value, to_value, Function as TeraFn, Result, Value};
use serde_derive::{Deserialize, Serialize};
use svg_metadata as svg;
use tera::{from_value, to_value, Error, Function as TeraFn, Result, Value};
use crate::global_fns::helpers::search_for_file; use crate::global_fns::helpers::search_for_file;
#[derive(Debug, Serialize, Deserialize)]
struct ResizeImageResponse {
/// The final URL for that asset
url: String,
/// The path to the static asset generated
static_path: String,
}
#[derive(Debug)] #[derive(Debug)]
pub struct ResizeImage { pub struct ResizeImage {
/// The base path of the Zola site /// The base path of the Zola site
@ -74,33 +62,11 @@ impl TeraFn for ResizeImage {
} }
}; };
let imageop = let response = imageproc
imageproc::ImageOp::from_args(path, file_path, &op, width, height, &format, quality) .enqueue(path, file_path, &op, width, height, &format, quality)
.map_err(|e| format!("`resize_image`: {}", e))?; .map_err(|e| format!("`resize_image`: {}", e))?;
let (static_path, url) = imageproc.insert(imageop);
to_value(ResizeImageResponse { to_value(response).map_err(Into::into)
static_path: static_path.to_string_lossy().into_owned(),
url,
})
.map_err(|err| err.into())
}
}
// Try to read the image dimensions for a given image
fn image_dimensions(path: &Path) -> Result<(u32, u32)> {
if let Some("svg") = path.extension().and_then(OsStr::to_str) {
let img = svg::Metadata::parse_file(&path)
.map_err(|e| Error::chain(format!("Failed to process SVG: {}", path.display()), e))?;
match (img.height(), img.width(), img.view_box()) {
(Some(h), Some(w), _) => Ok((h as u32, w as u32)),
(_, _, Some(view_box)) => Ok((view_box.height as u32, view_box.width as u32)),
_ => Err("Invalid dimensions: SVG width/height and viewbox not set.".into()),
}
} else {
let img = image::open(&path)
.map_err(|e| Error::chain(format!("Failed to process image: {}", path.display()), e))?;
Ok((img.height(), img.width()))
} }
} }
@ -139,11 +105,10 @@ impl TeraFn for GetImageMetadata {
return Err(format!("`resize_image`: Cannot find path: {}", path).into()); return Err(format!("`resize_image`: Cannot find path: {}", path).into());
} }
}; };
let (height, width) = image_dimensions(&src_path)?;
let mut map = tera::Map::new(); let response = imageproc::read_image_metadata(&src_path)
map.insert(String::from("height"), Value::Number(tera::Number::from(height))); .map_err(|e| format!("`resize_image`: {}", e))?;
map.insert(String::from("width"), Value::Number(tera::Number::from(width))); to_value(response).map_err(Into::into)
Ok(Value::Object(map))
} }
} }
@ -190,13 +155,15 @@ mod tests {
let data = static_fn.call(&args).unwrap().as_object().unwrap().clone(); let data = static_fn.call(&args).unwrap().as_object().unwrap().clone();
let static_path = Path::new("static").join("processed_images"); let static_path = Path::new("static").join("processed_images");
// TODO: Use `assert_processed_path_matches()` from imageproc so that hashes don't need to be hardcoded
assert_eq!( assert_eq!(
data["static_path"], data["static_path"],
to_value(&format!("{}", static_path.join("e49f5bd23ec5007c00.jpg").display())).unwrap() to_value(&format!("{}", static_path.join("6a89d6483cdc5f7700.jpg").display())).unwrap()
); );
assert_eq!( assert_eq!(
data["url"], data["url"],
to_value("http://a-website.com/processed_images/e49f5bd23ec5007c00.jpg").unwrap() to_value("http://a-website.com/processed_images/6a89d6483cdc5f7700.jpg").unwrap()
); );
// 2. resizing an image in content with a relative path // 2. resizing an image in content with a relative path
@ -204,33 +171,33 @@ mod tests {
let data = static_fn.call(&args).unwrap().as_object().unwrap().clone(); let data = static_fn.call(&args).unwrap().as_object().unwrap().clone();
assert_eq!( assert_eq!(
data["static_path"], data["static_path"],
to_value(&format!("{}", static_path.join("32454a1e0243976c00.jpg").display())).unwrap() to_value(&format!("{}", static_path.join("202d9263f4dbc95900.jpg").display())).unwrap()
); );
assert_eq!( assert_eq!(
data["url"], data["url"],
to_value("http://a-website.com/processed_images/32454a1e0243976c00.jpg").unwrap() to_value("http://a-website.com/processed_images/202d9263f4dbc95900.jpg").unwrap()
); );
// 3. resizing with an absolute path is the same as the above // 3. resizing with an absolute path is the same as the above
args.insert("path".to_string(), to_value("/content/gutenberg.jpg").unwrap()); args.insert("path".to_string(), to_value("/content/gutenberg.jpg").unwrap());
assert_eq!( assert_eq!(
data["static_path"], data["static_path"],
to_value(&format!("{}", static_path.join("32454a1e0243976c00.jpg").display())).unwrap() to_value(&format!("{}", static_path.join("202d9263f4dbc95900.jpg").display())).unwrap()
); );
assert_eq!( assert_eq!(
data["url"], data["url"],
to_value("http://a-website.com/processed_images/32454a1e0243976c00.jpg").unwrap() to_value("http://a-website.com/processed_images/202d9263f4dbc95900.jpg").unwrap()
); );
// 4. resizing an image in content starting with `@/` is the same as 2 and 3 // 4. resizing an image in content starting with `@/` is the same as 2 and 3
args.insert("path".to_string(), to_value("@/gutenberg.jpg").unwrap()); args.insert("path".to_string(), to_value("@/gutenberg.jpg").unwrap());
assert_eq!( assert_eq!(
data["static_path"], data["static_path"],
to_value(&format!("{}", static_path.join("32454a1e0243976c00.jpg").display())).unwrap() to_value(&format!("{}", static_path.join("202d9263f4dbc95900.jpg").display())).unwrap()
); );
assert_eq!( assert_eq!(
data["url"], data["url"],
to_value("http://a-website.com/processed_images/32454a1e0243976c00.jpg").unwrap() to_value("http://a-website.com/processed_images/202d9263f4dbc95900.jpg").unwrap()
); );
// 5. resizing an image with a relative path not starting with static or content // 5. resizing an image with a relative path not starting with static or content
@ -238,11 +205,11 @@ mod tests {
let data = static_fn.call(&args).unwrap().as_object().unwrap().clone(); let data = static_fn.call(&args).unwrap().as_object().unwrap().clone();
assert_eq!( assert_eq!(
data["static_path"], data["static_path"],
to_value(&format!("{}", static_path.join("c8aaba7b0593a60b00.jpg").display())).unwrap() to_value(&format!("{}", static_path.join("6296a3c153f701be00.jpg").display())).unwrap()
); );
assert_eq!( assert_eq!(
data["url"], data["url"],
to_value("http://a-website.com/processed_images/c8aaba7b0593a60b00.jpg").unwrap() to_value("http://a-website.com/processed_images/6296a3c153f701be00.jpg").unwrap()
); );
} }