fix statuses being deleted from threads

routes/_actions/deleteStatuses.js

@@ -35,9 +35,7 @@ async function doDeleteStatus (instanceName, statusId) {
   let rebloggedIds = await getIdsThatRebloggedThisStatus(instanceName, statusId)
   let statusIdsToDelete = Array.from(new Set([statusId].concat(rebloggedIds).filter(Boolean)))
   let notificationIdsToDelete = new Set(await getNotificationIdsForStatuses(instanceName, statusIdsToDelete))
-  await Promise.all([
+  await deleteStatusesAndNotifications(instanceName, statusIdsToDelete, notificationIdsToDelete)
-    deleteStatusesAndNotifications(instanceName, statusIdsToDelete, notificationIdsToDelete)
-  ])
 }
 
 export function deleteStatus (instanceName, statusId) {

routes/_database/cleanup.js

@@ -14,6 +14,8 @@ import {
 import debounce from 'lodash/debounce'
 import { store } from '../_store/store'
 import { mark, stop } from '../_utils/marks'
+import { deleteAll } from './utils'
+import { createPinnedStatusKeyRange, createThreadKeyRange } from './keys'
 
 const BATCH_SIZE = 20
 const TIME_AGO = 14 * 24 * 60 * 60 * 1000 // two weeks ago
@@ -34,18 +36,20 @@ function batchedGetAll (callGetAll, callback) {
 
 function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cutoff) {
   batchedGetAll(
-    () => statusesStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
+    () => statusesStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
     results => {
-      results.forEach(result => {
+      results.forEach(statusId => {
-        statusesStore.delete(result.id)
+        statusesStore.delete(statusId)
-        threadsStore.delete(result.id)
+        deleteAll(
-        let req = statusTimelinesStore.index('statusId').getAllKeys(IDBKeyRange.only(result.id))
+          statusTimelinesStore,
-        req.onsuccess = e => {
+          statusTimelinesStore.index('statusId'),
-          let keys = e.target.result
+          IDBKeyRange.only(statusId)
-          keys.forEach(key => {
+        )
-            statusTimelinesStore.delete(key)
+        deleteAll(
-          })
+          threadsStore,
-        }
+          threadsStore,
+          createThreadKeyRange(statusId)
+        )
       })
     }
   )
@@ -53,17 +57,15 @@ function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cut
 
 function cleanupNotifications (notificationsStore, notificationTimelinesStore, cutoff) {
   batchedGetAll(
-    () => notificationsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
+    () => notificationsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
     results => {
-      results.forEach(result => {
+      results.forEach(notificationId => {
-        notificationsStore.delete(result.id)
+        notificationsStore.delete(notificationId)
-        let req = notificationTimelinesStore.index('notificationId').getAllKeys(IDBKeyRange.only(result.id))
+        deleteAll(
-        req.onsuccess = e => {
+          notificationTimelinesStore,
-          let keys = e.target.result
+          notificationTimelinesStore.index('notificationId'),
-          keys.forEach(key => {
+          IDBKeyRange.only(notificationId)
-            notificationTimelinesStore.delete(key)
+        )
-          })
-        }
       })
     }
   )
@@ -71,18 +73,15 @@ function cleanupNotifications (notificationsStore, notificationTimelinesStore, c
 
 function cleanupAccounts (accountsStore, pinnedStatusesStore, cutoff) {
   batchedGetAll(
-    () => accountsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
+    () => accountsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
-    (results) => {
+    results => {
-      results.forEach(result => {
+      results.forEach(accountId => {
-        accountsStore.delete(result.id)
+        accountsStore.delete(accountId)
-        let keyRange = IDBKeyRange.bound(result.id + '\u0000', result.id + '\u0000\uffff')
+        deleteAll(
-        let req = pinnedStatusesStore.getAllKeys(keyRange)
+          pinnedStatusesStore,
-        req.onsuccess = e => {
+          pinnedStatusesStore,
-          let keys = e.target.result
+          createPinnedStatusKeyRange(accountId)
-          keys.forEach(key => {
+        )
-            pinnedStatusesStore.delete(key)
-          })
-        }
       })
     }
   )
@@ -90,10 +89,10 @@ function cleanupAccounts (accountsStore, pinnedStatusesStore, cutoff) {
 
 function cleanupRelationships (relationshipsStore, cutoff) {
   batchedGetAll(
-    () => relationshipsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
+    () => relationshipsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
-    (results) => {
+    results => {
-      results.forEach(result => {
+      results.forEach(relationshipId => {
-        relationshipsStore.delete(result.id)
+        relationshipsStore.delete(relationshipId)
       })
     }
   )

routes/_database/constants.js

@@ -1,12 +1,12 @@
-export const STATUSES_STORE = 'statuses-v1'
+export const STATUSES_STORE = 'statuses-v2'
-export const STATUS_TIMELINES_STORE = 'status_timelines-v1'
+export const STATUS_TIMELINES_STORE = 'status_timelines-v2'
-export const META_STORE = 'meta-v1'
+export const META_STORE = 'meta-v2'
-export const ACCOUNTS_STORE = 'accounts-v1'
+export const ACCOUNTS_STORE = 'accounts-v2'
-export const RELATIONSHIPS_STORE = 'relationships-v1'
+export const RELATIONSHIPS_STORE = 'relationships-v2'
-export const NOTIFICATIONS_STORE = 'notifications-v1'
+export const NOTIFICATIONS_STORE = 'notifications-v2'
-export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines-v1'
+export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines-v2'
-export const PINNED_STATUSES_STORE = 'pinned_statuses-v1'
+export const PINNED_STATUSES_STORE = 'pinned_statuses-v2'
-export const THREADS_STORE = 'threads-v1'
+export const THREADS_STORE = 'threads-v2'
 
 export const TIMESTAMP = '__pinafore_ts'
 export const ACCOUNT_ID = '__pinafore_acct_id'

routes/_database/databaseLifecycle.js

@@ -13,10 +13,12 @@ import {
   STATUS_ID
 } from './constants'
 
+import forEach from 'lodash/forEach'
+
 const openReqs = {}
 const databaseCache = {}
 
-const DB_VERSION = 6
+const DB_VERSION = 7
 
 export function getDatabase (instanceName) {
   if (!instanceName) {
@@ -35,28 +37,46 @@ export function getDatabase (instanceName) {
     }
     req.onupgradeneeded = (e) => {
       let db = req.result
-      let tx = e.currentTarget.transaction
+
-      if (e.oldVersion < 5) {
+      function createObjectStore (name, init, indexes) {
-        db.createObjectStore(STATUSES_STORE, {keyPath: 'id'})
+        let store = init
-          .createIndex(TIMESTAMP, TIMESTAMP)
+          ? db.createObjectStore(name, init)
-        db.createObjectStore(STATUS_TIMELINES_STORE)
+          : db.createObjectStore(name)
-          .createIndex('statusId', '')
+        if (indexes) {
-        db.createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'})
+          forEach(indexes, (indexValue, indexKey) => {
-          .createIndex(TIMESTAMP, TIMESTAMP)
+            store.createIndex(indexKey, indexValue)
-        db.createObjectStore(NOTIFICATION_TIMELINES_STORE)
+          })
-          .createIndex('notificationId', '')
-        db.createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'})
-          .createIndex(TIMESTAMP, TIMESTAMP)
-        db.createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'})
-          .createIndex(TIMESTAMP, TIMESTAMP)
-        db.createObjectStore(META_STORE)
-        db.createObjectStore(PINNED_STATUSES_STORE)
-        tx.objectStore(STATUSES_STORE).createIndex(REBLOG_ID, REBLOG_ID)
-        tx.objectStore(NOTIFICATIONS_STORE).createIndex('statusId', 'statusId')
-        db.createObjectStore(THREADS_STORE)
         }
-      if (e.oldVersion < 6) {
+      }
-        tx.objectStore(NOTIFICATIONS_STORE).createIndex(STATUS_ID, STATUS_ID)
+
+      if (e.oldVersion < 7) {
+        createObjectStore(STATUSES_STORE, {keyPath: 'id'}, {
+          [TIMESTAMP]: TIMESTAMP,
+          [REBLOG_ID]: REBLOG_ID
+        })
+        createObjectStore(STATUS_TIMELINES_STORE, null, {
+          'statusId': ''
+        })
+        createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'}, {
+          [TIMESTAMP]: TIMESTAMP,
+          [STATUS_ID]: STATUS_ID
+        })
+        createObjectStore(NOTIFICATION_TIMELINES_STORE, null, {
+          'notificationId': ''
+        })
+        createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'}, {
+          [TIMESTAMP]: TIMESTAMP
+        })
+        createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'}, {
+          [TIMESTAMP]: TIMESTAMP
+        })
+        createObjectStore(THREADS_STORE, null, {
+          'statusId': ''
+        })
+        createObjectStore(PINNED_STATUSES_STORE, null, {
+          'statusId': ''
+        })
+        createObjectStore(META_STORE)
       }
     }
     req.onsuccess = () => resolve(req.result)

routes/_database/keys.js

@@ -0,0 +1,47 @@
+import { toReversePaddedBigInt, zeroPad } from '../_utils/sorting'
+
+//
+// timelines
+//
+
+export function createTimelineId (timeline, id) {
+  // reverse chronological order, prefixed by timeline
+  return timeline + '\u0000' + toReversePaddedBigInt(id)
+}
+
+export function createTimelineKeyRange (timeline, maxId) {
+  let negBigInt = maxId && toReversePaddedBigInt(maxId)
+  let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000')
+  let end = timeline + '\u0000\uffff'
+  return IDBKeyRange.bound(start, end, true, true)
+}
+
+//
+// threads
+//
+
+export function createThreadId (statusId, i) {
+  return statusId + '\u0000' + zeroPad(i, 5)
+}
+
+export function createThreadKeyRange (statusId) {
+  return IDBKeyRange.bound(
+    statusId + '\u0000',
+    statusId + '\u0000\uffff'
+  )
+}
+
+//
+// pinned statues
+//
+
+export function createPinnedStatusId (accountId, i) {
+  return accountId + '\u0000' + zeroPad(i, 3)
+}
+
+export function createPinnedStatusKeyRange (accountId) {
+  return IDBKeyRange.bound(
+    accountId + '\u0000',
+    accountId + '\u0000\uffff'
+  )
+}

routes/_database/timelines.js

@@ -1,4 +1,5 @@
-import { toPaddedBigInt, toReversePaddedBigInt } from '../_utils/sorting'
+import difference from 'lodash/difference'
+import times from 'lodash/times'
 import { cloneForStorage } from './helpers'
 import { dbPromise, getDatabase } from './databaseLifecycle'
 import {
@@ -16,13 +17,15 @@ import {
   REBLOG_ID,
   STATUS_ID, THREADS_STORE
 } from './constants'
-
+import {
-function createTimelineKeyRange (timeline, maxId) {
+  createThreadKeyRange,
-  let negBigInt = maxId && toReversePaddedBigInt(maxId)
+  createTimelineKeyRange,
-  let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000')
+  createTimelineId,
-  let end = timeline + '\u0000\uffff'
+  createThreadId,
-  return IDBKeyRange.bound(start, end, true, true)
+  createPinnedStatusKeyRange,
-}
+  createPinnedStatusId
+} from './keys'
+import { deleteAll } from './utils'
 
 function cacheStatus (status, instanceName) {
   setInCache(statusesCache, instanceName, status.id, status)
@@ -80,7 +83,8 @@ async function getStatusThread (instanceName, statusId) {
   const db = await getDatabase(instanceName)
   return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
     let [ threadsStore, statusesStore, accountsStore ] = stores
-    threadsStore.get(statusId).onsuccess = e => {
+    let keyRange = createThreadKeyRange(statusId)
+    threadsStore.getAll(keyRange).onsuccess = e => {
       let thread = e.target.result
       let res = new Array(thread.length)
       thread.forEach((otherStatusId, i) => {
@@ -177,11 +181,6 @@ function fetchNotification (notificationsStore, statusesStore, accountsStore, id
   }
 }
 
-function createTimelineId (timeline, id) {
-  // reverse chronological order, prefixed by timeline
-  return timeline + '\u0000' + toReversePaddedBigInt(id)
-}
-
 async function insertTimelineNotifications (instanceName, timeline, notifications) {
   for (let notification of notifications) {
     setInCache(notificationsCache, instanceName, notification.id, notification)
@@ -224,10 +223,18 @@ async function insertStatusThread (instanceName, statusId, statuses) {
   let storeNames = [THREADS_STORE, STATUSES_STORE, ACCOUNTS_STORE]
   await dbPromise(db, storeNames, 'readwrite', (stores) => {
     let [ threadsStore, statusesStore, accountsStore ] = stores
-    threadsStore.put(statuses.map(_ => _.id), statusId)
+    threadsStore.getAllKeys(createThreadKeyRange(statusId)).onsuccess = e => {
-    for (let status of statuses) {
+      let existingKeys = e.target.result
-      storeStatus(statusesStore, accountsStore, status)
+      let newKeys = times(statuses.length, i => createThreadId(statusId, i))
+      let keysToDelete = difference(existingKeys, newKeys)
+      for (let key of keysToDelete) {
+        threadsStore.delete(key)
       }
+    }
+    statuses.forEach((otherStatus, i) => {
+      storeStatus(statusesStore, accountsStore, otherStatus)
+      threadsStore.put(otherStatus.id, createThreadId(statusId, i))
+    })
   })
 }
 
@@ -323,7 +330,8 @@ export async function deleteStatusesAndNotifications (instanceName, statusIds, n
     STATUS_TIMELINES_STORE,
     NOTIFICATIONS_STORE,
     NOTIFICATION_TIMELINES_STORE,
-    PINNED_STATUSES_STORE
+    PINNED_STATUSES_STORE,
+    THREADS_STORE
   ]
   await dbPromise(db, storeNames, 'readwrite', (stores) => {
     let [
@@ -331,33 +339,41 @@ export async function deleteStatusesAndNotifications (instanceName, statusIds, n
       statusTimelinesStore,
       notificationsStore,
       notificationTimelinesStore,
-      pinnedStatusesStore
+      pinnedStatusesStore,
+      threadsStore
     ] = stores
 
     function deleteStatus (statusId) {
-      pinnedStatusesStore.delete(statusId).onerror = e => {
-        e.preventDefault()
-        e.stopPropagation()
-      }
       statusesStore.delete(statusId)
-      let getAllReq = statusTimelinesStore.index('statusId')
+      deleteAll(
-        .getAllKeys(IDBKeyRange.only(statusId))
+        pinnedStatusesStore,
-      getAllReq.onsuccess = e => {
+        pinnedStatusesStore.index('statusId'),
-        for (let result of e.target.result) {
+        IDBKeyRange.only(statusId)
-          statusTimelinesStore.delete(result)
+      )
-        }
+      deleteAll(
-      }
+        statusTimelinesStore,
+        statusTimelinesStore.index('statusId'),
+        IDBKeyRange.only(statusId)
+      )
+      deleteAll(
+        threadsStore,
+        threadsStore.index('statusId'),
+        IDBKeyRange.only(statusId)
+      )
+      deleteAll(
+        threadsStore,
+        threadsStore,
+        createThreadKeyRange(statusId)
+      )
     }
 
     function deleteNotification (notificationId) {
       notificationsStore.delete(notificationId)
-      let getAllReq = notificationTimelinesStore.index('statusId')
+      deleteAll(
-        .getAllKeys(IDBKeyRange.only(notificationId))
+        notificationTimelinesStore,
-      getAllReq.onsuccess = e => {
+        notificationTimelinesStore.index('statusId'),
-        for (let result of e.target.result) {
+        IDBKeyRange.only(notificationId)
-          notificationTimelinesStore.delete(result)
+      )
-        }
-      }
     }
 
     for (let statusId of statusIds) {
@@ -383,7 +399,7 @@ export async function insertPinnedStatuses (instanceName, accountId, statuses) {
     let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores
     statuses.forEach((status, i) => {
       storeStatus(statusesStore, accountsStore, status)
-      pinnedStatusesStore.put(status.id, accountId + '\u0000' + toPaddedBigInt(i))
+      pinnedStatusesStore.put(status.id, createPinnedStatusId(accountId, i))
     })
   })
 }
@@ -393,10 +409,7 @@ export async function getPinnedStatuses (instanceName, accountId) {
   const db = await getDatabase(instanceName)
   return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
     let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores
-    let keyRange = IDBKeyRange.bound(
+    let keyRange = createPinnedStatusKeyRange(accountId)
-      accountId + '\u0000',
-      accountId + '\u0000\uffff'
-    )
     pinnedStatusesStore.getAll(keyRange).onsuccess = e => {
       let pinnedResults = e.target.result
       let res = new Array(pinnedResults.length)
@@ -410,19 +423,6 @@ export async function getPinnedStatuses (instanceName, accountId) {
   })
 }
 
-//
-// notifications by status
-//
-
-export async function getNotificationIdsForStatus (instanceName, statusId) {
-  const db = await getDatabase(instanceName)
-  return dbPromise(db, NOTIFICATIONS_STORE, 'readonly', (notificationStore, callback) => {
-    notificationStore.index(statusId).getAllKeys(IDBKeyRange.only(statusId)).onsuccess = e => {
-      callback(Array.from(e.target.result))
-    }
-  })
-}
-
 //
 // update statuses
 //

routes/_database/utils.js

@@ -0,0 +1,7 @@
+export function deleteAll (store, index, keyRange) {
+  index.getAllKeys(keyRange).onsuccess = e => {
+    for (let result of e.target.result) {
+      store.delete(result)
+    }
+  }
+}

routes/_utils/sorting.js

@@ -1,7 +1,11 @@
 import padStart from 'lodash/padStart'
 
+export function zeroPad (str, toSize) {
+  return padStart(str, toSize, '0')
+}
+
 export function toPaddedBigInt (id) {
-  return padStart(id, 30, '0')
+  return zeroPad(id, 30)
 }
 
 export function toReversePaddedBigInt (id) {

tests/serverActions.js

@@ -20,6 +20,11 @@ export async function postAsAdmin (text) {
     null, null, false, null, 'public')
 }
 
+export async function postReplyAsAdmin (text, inReplyTo) {
+  return postStatus(instanceName, users.admin.accessToken, text,
+    inReplyTo, null, false, null, 'public')
+}
+
 export async function deleteAsAdmin (statusId) {
   return deleteStatus(instanceName, users.admin.accessToken, statusId)
 }

tests/spec/105-deletes.js

@@ -1,6 +1,9 @@
 import { foobarRole } from '../roles'
-import { getNthStatus } from '../utils'
+import {
-import { deleteAsAdmin, postAsAdmin } from '../serverActions'
+  clickToNotificationsAndBackHome, forceOffline, forceOnline, getNthStatus, getUrl, homeNavButton,
+  sleep
+} from '../utils'
+import { deleteAsAdmin, postAsAdmin, postReplyAsAdmin } from '../serverActions'
 
 fixture`105-deletes.js`
   .page`http://localhost:4002`
@@ -9,7 +12,44 @@ test('deleted statuses are removed from the timeline', async t => {
   await t.useRole(foobarRole)
     .hover(getNthStatus(0))
   let status = await postAsAdmin("I'm gonna delete this")
+  await sleep(1000)
   await t.expect(getNthStatus(0).innerText).contains("I'm gonna delete this")
   await deleteAsAdmin(status.id)
+  await sleep(1000)
   await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
+  await clickToNotificationsAndBackHome(t)
+  await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
+  await t.navigateTo('/notifications')
+  await forceOffline()
+  await t.click(homeNavButton)
+  await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
+  await forceOnline()
+  await t
+    .navigateTo('/')
+    .expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
+})
+
+test('deleted statuses are removed from threads', async t => {
+  await t.useRole(foobarRole)
+    .hover(getNthStatus(0))
+  let status = await postAsAdmin("I won't delete this")
+  let reply = await postReplyAsAdmin('But I will delete this', status.id)
+  await sleep(5000)
+  await t.expect(getNthStatus(0).innerText).contains('But I will delete this')
+    .expect(getNthStatus(1).innerText).contains("I won't delete this")
+    .click(getNthStatus(1))
+    .expect(getUrl()).contains('/statuses')
+    .expect(getNthStatus(0).innerText).contains("I won't delete this")
+    .expect(getNthStatus(1).innerText).contains('But I will delete this')
+  await deleteAsAdmin(reply.id)
+  await sleep(1000)
+  await t.expect(getNthStatus(1).exists).notOk()
+    .expect(getNthStatus(0).innerText).contains("I won't delete this")
+  await t.navigateTo('/')
+  await forceOffline()
+  await t.click(getNthStatus(0))
+    .expect(getUrl()).contains('/statuses')
+    .expect(getNthStatus(1).exists).notOk()
+    .expect(getNthStatus(0).innerText).contains("I won't delete this")
+  await forceOnline()
 })

tests/utils.js

@@ -186,3 +186,10 @@ export async function scrollToStatus (t, n) {
   }
   await t.hover(getNthStatus(n))
 }
+
+export async function clickToNotificationsAndBackHome (t) {
+  await t.click(notificationsNavButton)
+    .expect(getUrl()).eql('http://localhost:4002/notifications')
+    .click(homeNavButton)
+    .expect(getUrl()).eql('http://localhost:4002/')
+}