feat(media): Blurhash (#1381)

* chore(npm): Install blurhash

* feat(media): Show blurhash

* fix(media/blurhash): Better sensitive video handling

* feat(media): Preference for using blurhash

* chore(utils/blurhash): Add performance marks

* fix(utils/blurhash): Performance marks

* fix(utils/blurhash): Use correct dimension

* refactor(utils/blurhash): Use constant for number of pixels

* refactor(media): Simplify logic for displaying blurhash

* chore(tests/spec): Attempt to adjust sensitivity tests for blurhash

* chore(tests/spec): Update sensitivity tests for blurhash

* chore(tests/spec): Check for sensitive

* fix(media/blurhash): Handle videos

* fix: Video handling

* fix: Videos

* minor refactoring, fix Svelte warning

* fix: Large inline images and videos

* feat(settings): Rename blurhash setting

* refactor: Use toBlob, block media rendering until blurhash ready

* refactor: Move computations to Web Worker

* fix(workers/blurhash): More error handling

* feat(workers/blurhash): Use quick-lru for caching

* fix: Don't create Context2D needlessly

* fix(workers/blurhash): Increase cache size to 100

* fix(workers/blurhash): Don't resolve promise twice

* fix(utils/decode-image): Ignore data URLs

Throws exception which prevents the image from loading.

package.json

@@ -48,6 +48,7 @@
     "@webcomponents/custom-elements": "^1.2.4",
     "babel-loader": "^8.0.6",
     "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
+    "blurhash": "^1.1.3",
     "cheerio": "^1.0.0-rc.2",
     "child-process-promise": "^2.2.1",
     "chokidar": "^3.0.1",
@@ -109,7 +110,8 @@
     "mocha": "^6.1.4",
     "now": "^15.7.0",
     "standard": "^13.1.0",
-    "testcafe": "^1.2.1"
+    "testcafe": "^1.2.1",
+    "worker-loader": "^2.0.0"
   },
   "engines": {
     "node": ">= 8"
@@ -150,7 +152,10 @@
       "customElements",
       "AbortController",
       "matchMedia",
-      "MessageChannel"
+      "MessageChannel",
+      "ImageData",
+      "OffscreenCanvas",
+      "postMessage"
     ],
     "ignore": [
       "dist",

src/routes/_components/LazyImage.html

@@ -53,7 +53,8 @@
       height: void 0,
       ariaHidden: false,
       alt: '',
-      title: ''
+      title: '',
+      blurhash: void 0
     }),
     computed: {
       computedStyle: ({ background }) => {
@@ -71,7 +72,7 @@
         return `object-position: ${coordsToPercent(focus.x)}% ${100 - coordsToPercent(focus.y)}%;`
       },
       fillFixSize: ({ forceSize, $largeInlineMedia }) => !$largeInlineMedia && !forceSize,
-      displaySrc: ({ error, src, fallback }) => ((error && fallback) || src)
+      displaySrc: ({ blurhash, error, src, fallback }) => (blurhash || (error && fallback) || src)
     }
   }
 </script>

src/routes/_components/NonAutoplayGifv.html

@@ -17,6 +17,7 @@
       alt={label || ''}
       title={label || ''}
       src={staticSrc}
+      blurhash={blurhash}
       fallback={oneTransparentPixel}
       {width}
       {height}
@@ -24,7 +25,9 @@
       {focus}
     />
   {/if}
+  {#if !blurhash}
     <PlayVideoIcon className={playing ? 'hidden' : ''}/>
+  {/if}
 </div>
 <style>
   .non-autoplay-gifv {

src/routes/_components/status/Media.html

@@ -1,4 +1,19 @@
 {#if type === 'video' || type === 'audio'}
+  {#if blurhash}
+    {#if type === 'video'}
+      <LazyImage
+        alt={description}
+        title={description}
+        src={previewUrl}
+        fallback={oneTransparentPixel}
+        blurhash={blurhash}
+        width={inlineWidth}
+        height={inlineHeight}
+        background="var(--loading-bg)"
+        {focus}
+      />
+    {/if}
+  {:else}
     <button id={elementId}
             type="button"
             class="play-video-button focus-after {$largeInlineMedia ? '' : 'fixed-size'} {type === 'audio' ? 'play-audio-button' : ''}"
@@ -11,6 +26,7 @@
           title={description}
           src={previewUrl}
           fallback={oneTransparentPixel}
+          blurhash={blurhash}
           width={inlineWidth}
           height={inlineHeight}
           background="var(--loading-bg)"
@@ -18,6 +34,7 @@
         />
       {/if}
     </button>
+  {/if}
 {:else}
   <button id={elementId}
           type="button"
@@ -40,6 +57,7 @@
       class={noNativeWidthHeight ? 'no-native-width-height' : ''}
       label="Animated GIF: {description}"
       poster={previewUrl}
+      blurhash={blurhash}
       src={url}
       staticSrc={previewUrl}
       width={inlineWidth}
@@ -53,6 +71,7 @@
       title={description}
       src={previewUrl}
       fallback={oneTransparentPixel}
+      blurhash={blurhash}
       width={inlineWidth}
       height={inlineHeight}
       background="var(--loading-bg)"
@@ -91,11 +110,16 @@
   import LazyImage from '../LazyImage.html'
   import AutoplayVideo from '../AutoplayVideo.html'
   import { registerClickDelegate } from '../../_utils/delegate'
+  import { decode } from '../../_utils/blurhash'
 
-  export default {
+export default {
-    oncreate () {
+    async oncreate () {
-      const { elementId } = this.get()
+      const { elementId, media } = this.get()
       registerClickDelegate(this, elementId, () => this.onClick())
+
+      if (media.blurhash) {
+        this.set({ decodedBlurhash: await decode(media.blurhash) })
+      }
     },
     computed: {
       focus: ({ meta }) => meta && meta.focus,
@@ -126,6 +150,7 @@
       elementId: ({ media, uuid }) => `media-${uuid}-${media.id}`,
       description: ({ media }) => media.description || '',
       previewUrl: ({ media }) => media.preview_url,
+      blurhash: ({ showBlurhash, decodedBlurhash }) => showBlurhash && decodedBlurhash,
       url: ({ media }) => media.url,
       type: ({ media }) => media.type
     },
@@ -141,6 +166,7 @@
     },
     data: () => ({
       oneTransparentPixel: ONE_TRANSPARENT_PIXEL,
+      decodedBlurhash: ONE_TRANSPARENT_PIXEL,
       mouseover: void 0
     }),
     store: () => store,

src/routes/_components/status/MediaAttachments.html

@@ -1,7 +1,7 @@
 <div class={computedClass}
      style="grid-template-columns: repeat({nCols}, 1fr);" >
   {#each mediaAttachments as media, index}
-    <Media {media} {uuid} {mediaAttachments} {index} />
+    <Media {media} {uuid} {mediaAttachments} {index} {showBlurhash} />
   {/each}
 </div>
 <style>
@@ -39,6 +39,7 @@
   }
 
   .status-media.status-media-is-sensitive {
+    height: inherit;
     margin: 0;
   }
 </style>
@@ -55,6 +56,10 @@
         twoCols && 'two-cols',
         !$largeInlineMedia && 'grouped-images'
       ),
+      showBlurhash:
+        ({ sensitive, sensitiveShown, mediaAttachments }) => {
+          return sensitive && mediaAttachments.every(attachment => !!attachment.blurhash) ? !sensitiveShown : false
+        },
       nCols:
         ({ mediaAttachments, $largeInlineMedia }) => {
           return (!$largeInlineMedia && mediaAttachments.length > 1) ? 2 : 1

src/routes/_components/status/StatusMediaAttachments.html

@@ -10,28 +10,32 @@
       <SvgIcon className="status-sensitive-media-svg" href="#fa-eye-slash" />
     </div>
   </button>
-  <MediaAttachments {mediaAttachments} {sensitive} {uuid} />
   {:else}
     <button id={elementId}
             type="button"
             class="status-sensitive-media-button"
             aria-label="Show sensitive media" >
 
-      <div class="status-sensitive-media-warning">
+      <div class="{customWarningClass}">
+        <div class="status-sensitive-media-warning-text">
           Sensitive content. Click to show.
         </div>
+      </div>
       <div class="svg-wrapper">
         <SvgIcon className="status-sensitive-media-svg" href="#fa-eye" />
       </div>
     </button>
   {/if}
+  {#if sensitiveShown || canUseBlurhash}
+    <MediaAttachments {mediaAttachments} {sensitive} {sensitiveShown} {uuid} />
+  {/if}
   </div>
 </div>
 {#if enableShortcuts}
   <Shortcut scope={shortcutScope} key="y" on:pressed="toggleSensitiveMedia()"/>
 {/if}
 {:else}
-  <MediaAttachments {mediaAttachments} {sensitive} {uuid} />
+  <MediaAttachments {mediaAttachments} {sensitive} {sensitiveShown} {uuid} />
 {/if}
 <style>
   .status-sensitive-media-container {
@@ -81,6 +85,7 @@
   }
 
   .status-sensitive-media-hidden .status-sensitive-media-button {
+    position: absolute;
     right: 0;
     bottom: 0;
     width: 100%;
@@ -96,7 +101,6 @@
   }
 
   .status-sensitive-media-container .status-sensitive-media-warning {
-    position: absolute;
     top: 0;
     left: 0;
     right: 0;
@@ -109,6 +113,21 @@
     padding: 0 10px;
   }
 
+  .status-sensitive-media-container .status-sensitive-media-warning-transparent {
+    position: absolute;
+  }
+
+  .status-sensitive-media-container .status-sensitive-media-warning-opaque {
+    background: var(--mask-bg);
+    height: 100%;
+  }
+
+  .status-sensitive-media-container .status-sensitive-media-warning-transparent .status-sensitive-media-warning-text {
+    background: var(--mask-bg);
+    padding: 10px;
+    border-radius: 6px;
+  }
+
   .status-sensitive-media-container .svg-wrapper {
     display: flex;
     align-items: flex-start;
@@ -119,6 +138,7 @@
   }
   .status-sensitive-media-hidden .svg-wrapper {
     position: absolute;
+    background: none;
     top: 0;
     left: 0;
     right: 0;
@@ -171,6 +191,7 @@
         $largeInlineMedia ? 'not-grouped-images' : 'grouped-images'
       ),
       mediaAttachments: ({ originalStatus }) => originalStatus.media_attachments,
+      canUseBlurhash: ({ $ignoreBlurhash, mediaAttachments }) => !$ignoreBlurhash && mediaAttachments && mediaAttachments.every(media => !!media.blurhash),
       sensitiveShown: ({ $sensitivesShown, uuid }) => !!$sensitivesShown[uuid],
       sensitive: ({ originalStatus, $markMediaAsSensitive, $neverMarkMediaAsSensitive }) => (
         !$neverMarkMediaAsSensitive && ($markMediaAsSensitive || originalStatus.sensitive)
@@ -181,7 +202,11 @@
           return ''
         }
         return `padding-bottom: ${Math.ceil(mediaAttachments.length / 2) * 29}%;`
-      }
+      },
+      customWarningClass: ({ canUseBlurhash }) => classname(
+        'status-sensitive-media-warning',
+        canUseBlurhash ? 'status-sensitive-media-warning-transparent' : 'status-sensitive-media-warning-opaque'
+      )
     },
     methods: {
       toggleSensitiveMedia () {

src/routes/_pages/settings/general.html

@@ -8,6 +8,11 @@
              bind:checked="$neverMarkMediaAsSensitive" on:change="onChange(event)">
       <label for="choice-never-mark-media-sensitive">Show sensitive media by default</label>
     </div>
+    <div class="setting-group">
+        <input type="checkbox" id="choice-use-blurhash"
+               bind:checked="$ignoreBlurhash" on:change="onChange(event)">
+        <label for="choice-use-blurhash">Show a plain gray color for sensitive media</label>
+      </div>
     <div class="setting-group">
       <input type="checkbox" id="choice-mark-media-sensitive"
              bind:checked="$markMediaAsSensitive" on:change="onChange(event)">

src/routes/_store/store.js

@@ -30,6 +30,7 @@ const persistedState = {
   loggedInInstancesInOrder: [],
   markMediaAsSensitive: false,
   neverMarkMediaAsSensitive: false,
+  ignoreBlurhash: false,
   omitEmojiInDisplayNames: undefined,
   pinnedPages: {},
   pushSubscriptions: {},

src/routes/_utils/blurhash.js

@@ -0,0 +1,51 @@
+import BlurhashWorker from 'worker-loader!../_workers/blurhash' // eslint-disable-line
+
+const RESOLUTION = 32
+let worker
+let canvas
+let canvasContext2D
+
+export function init () {
+  worker = worker || new BlurhashWorker()
+}
+
+export async function decode (blurhash) {
+  return new Promise((resolve, reject) => {
+    try {
+      init()
+
+      const onMessage = ({ data: { encoded, decoded, imageData, error } }) => {
+        if (encoded !== blurhash) {
+          return
+        }
+
+        worker.removeEventListener('message', onMessage)
+
+        if (error) {
+          return reject(error)
+        }
+
+        if (decoded) {
+          resolve(decoded)
+        } else {
+          if (!canvas) {
+            canvas = document.createElement('canvas')
+            canvas.height = RESOLUTION
+            canvas.width = RESOLUTION
+            canvasContext2D = canvas.getContext('2d')
+          }
+
+          canvasContext2D.putImageData(imageData, 0, 0)
+          canvas.toBlob(blob => {
+            resolve(URL.createObjectURL(blob))
+          })
+        }
+      }
+
+      worker.addEventListener('message', onMessage)
+      worker.postMessage({ encoded: blurhash })
+    } catch (e) {
+      reject(e)
+    }
+  })
+}

src/routes/_utils/decodeImage.js

@@ -4,7 +4,7 @@ export function decodeImage (img) {
   // Remove this UA sniff when the Firefox bug is fixed
   // https://github.com/nolanlawson/pinafore/issues/1344#issuecomment-514312672
   // https://bugzilla.mozilla.org/show_bug.cgi?id=1565542
-  if (!IS_FIREFOX && typeof img.decode === 'function') {
+  if (!IS_FIREFOX && typeof img.decode === 'function' && !img.src.startsWith('data:image/png;base64,')) {
     return img.decode()
   }
 

src/routes/_workers/blurhash.js

@@ -0,0 +1,45 @@
+import { decode as decodeBlurHash } from 'blurhash'
+import QuickLRU from 'quick-lru'
+
+const RESOLUTION = 32
+const OFFSCREEN_CANVAS = typeof OffscreenCanvas === 'function'
+  ? new OffscreenCanvas(RESOLUTION, RESOLUTION) : null
+const OFFSCREEN_CANVAS_CONTEXT_2D = OFFSCREEN_CANVAS
+  ? OFFSCREEN_CANVAS.getContext('2d') : null
+const CACHE = new QuickLRU({ maxSize: 100 })
+
+self.addEventListener('message', ({ data: { encoded } }) => {
+  try {
+    if (CACHE.has(encoded)) {
+      if (OFFSCREEN_CANVAS) {
+        postMessage({ encoded, decoded: CACHE.get(encoded), imageData: null, error: null })
+      } else {
+        postMessage({ encoded, imageData: CACHE.get(encoded), decoded: null, error: null })
+      }
+    } else {
+      const pixels = decodeBlurHash(encoded, RESOLUTION, RESOLUTION)
+
+      if (pixels) {
+        const imageData = new ImageData(pixels, RESOLUTION, RESOLUTION)
+
+        if (OFFSCREEN_CANVAS) {
+          OFFSCREEN_CANVAS_CONTEXT_2D.putImageData(imageData, 0, 0)
+          OFFSCREEN_CANVAS.convertToBlob().then(blob => {
+            const decoded = URL.createObjectURL(blob)
+            CACHE.set(encoded, decoded)
+            postMessage({ encoded, decoded, imageData: null, error: null })
+          }).catch(error => {
+            postMessage({ encoded, decoded: null, imageData: null, error })
+          })
+        } else {
+          CACHE.set(encoded, imageData)
+          postMessage({ encoded, imageData, decoded: null, error: null })
+        }
+      } else {
+        postMessage({ encoded, decoded: null, imageData: null, error: new Error('decode did not return any pixels') })
+      }
+    }
+  } catch (error) {
+    postMessage({ encoded, decoded: null, imageData: null, error })
+  }
+})

tests/spec/008-status-media.js

@@ -13,10 +13,10 @@ test('shows sensitive images and videos', async t => {
   const videoIdx = homeTimeline.findIndex(_ => _.content === 'secret video')
 
   await scrollToStatus(t, 1 + kittenIdx)
-  await t.expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).exists).notOk()
+  await t.expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).getAttribute('src')).match(/^blob:http:\/\/localhost/)
     .click($(`${getNthStatusSelector(1 + kittenIdx)} .status-sensitive-media-button`))
     .expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).getAttribute('alt')).eql('kitten')
-    .expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).hasAttribute('src')).ok()
+    .expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).getAttribute('src')).match(/^http:\/\//)
     .hover(getNthStatus(1 + videoIdx))
     .expect($(`${getNthStatusSelector(1 + videoIdx)} .status-media .play-video-button`).exists).notOk()
     .click($(`${getNthStatusSelector(1 + videoIdx)} .status-sensitive-media-button`))

tests/spec/023-mark-media-as-sensitive.js

@@ -1,7 +1,7 @@
 import { loginAsFoobar } from '../roles'
 import {
   generalSettingsButton,
-  getNthStatus, getNthStatusMedia, getNthStatusSensitiveMediaButton, homeNavButton, markMediaSensitiveInput,
+  getNthStatus, getNthStatusMediaImg, getNthStatusSensitiveMediaButton, homeNavButton, markMediaSensitiveInput,
   scrollToStatus, settingsNavButton, neverMarkMediaSensitiveInput
 } from '../utils'
 
@@ -14,11 +14,11 @@ async function checkSensitivityForStatus (t, idx, sensitive) {
   if (sensitive) {
     await t
       .expect(getNthStatusSensitiveMediaButton(1 + idx).exists).ok()
-      .expect(getNthStatusMedia(1 + idx).exists).notOk()
+      .expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^blob:http:\/\/localhost/)
   } else {
     await t
       .expect(getNthStatusSensitiveMediaButton(1 + idx).exists).notOk()
-      .expect(getNthStatusMedia(1 + idx).exists).ok()
+      .expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^http:\/\//)
   }
 }
 

tests/spec/025-shortcuts-status.js

@@ -4,7 +4,7 @@ import {
   getNthFavorited,
   getNthStatus,
   getNthStatusContent,
-  getNthStatusMedia,
+  getNthStatusMediaImg,
   getNthStatusSensitiveMediaButton,
   getNthStatusSpoiler,
   getUrl, modalDialog,
@@ -104,11 +104,11 @@ test('Shortcut y shows/hides sensitive image', async t => {
   await t
     .expect(isNthStatusActive(1 + idx)()).ok()
     .expect(getNthStatusSensitiveMediaButton(1 + idx).exists).ok()
-    .expect(getNthStatusMedia(1 + idx).exists).notOk()
+    .expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^blob:http:\/\/localhost/)
     .pressKey('y')
-    .expect(getNthStatusMedia(1 + idx).exists).ok()
+    .expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^http:\/\//)
     .pressKey('y')
-    .expect(getNthStatusMedia(1 + idx).exists).notOk()
+    .expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^blob:http:\/\/localhost/)
 })
 
 test('Shortcut f toggles favorite status', async t => {

yarn.lock

@@ -1489,6 +1489,11 @@ bluebird@^3.5.5:
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.5.tgz#a8d0afd73251effbbd5fe384a77d73003c17a71f"
   integrity sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==
 
+blurhash@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.3.tgz#dc325af7da836d07a0861d830bdd63694382483e"
+  integrity sha512-yUhPJvXexbqbyijCIE/T2NCXcj9iNPhWmOKbPTuR/cm7Q5snXYIfnVnz6m7MWOXxODMz/Cr3UcVkRdHiuDVRDw==
+
 bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.4.0:
   version "4.11.8"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f"
@@ -4602,7 +4607,7 @@ loader-runner@^2.4.0:
   resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357"
   integrity sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==
 
-loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
+loader-utils@^1.0.0, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7"
   integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==
@@ -6600,6 +6605,14 @@ sax@^1.2.4, sax@~1.2.4:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
 
+schema-utils@^0.4.0:
+  version "0.4.7"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187"
+  integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==
+  dependencies:
+    ajv "^6.1.0"
+    ajv-keywords "^3.1.0"
+
 schema-utils@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770"
@@ -8044,6 +8057,14 @@ worker-farm@^1.7.0:
   dependencies:
     errno "~0.1.7"
 
+worker-loader@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-2.0.0.tgz#45fda3ef76aca815771a89107399ee4119b430ac"
+  integrity sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==
+  dependencies:
+    loader-utils "^1.0.0"
+    schema-utils "^0.4.0"
+
 wrap-ansi@^2.0.0, wrap-ansi@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"