data.coop/pinafore
8
0
Fork 0
Code Issues 1 Pull requests Packages Projects Releases Wiki Activity

fix statuses being deleted from threads

Browse source
This commit is contained in:
Nolan Lawson 2018-03-10 20:24:07 -08:00
parent 23a247a8c2
commit da2daa955d
11 changed files with 256 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
routes/_actions/deleteStatuses.js
View file

@ -35,9 +35,7 @@ async function doDeleteStatus (instanceName, statusId) {
let rebloggedIds = await getIdsThatRebloggedThisStatus(instanceName, statusId)
let statusIdsToDelete = Array.from(new Set([statusId].concat(rebloggedIds).filter(Boolean)))
let notificationIdsToDelete = new Set(await getNotificationIdsForStatuses(instanceName, statusIdsToDelete))
await Promise.all([
deleteStatusesAndNotifications(instanceName, statusIdsToDelete, notificationIdsToDelete)
])
await deleteStatusesAndNotifications(instanceName, statusIdsToDelete, notificationIdsToDelete)
}
export function deleteStatus (instanceName, statusId) {

73
routes/_database/cleanup.js
View file

@ -14,6 +14,8 @@ import {
import debounce from 'lodash/debounce'
import { store } from '../_store/store'
import { mark, stop } from '../_utils/marks'
import { deleteAll } from './utils'
import { createPinnedStatusKeyRange, createThreadKeyRange } from './keys'
const BATCH_SIZE = 20
const TIME_AGO = 14 * 24 * 60 * 60 * 1000 // two weeks ago
@ -34,18 +36,20 @@ function batchedGetAll (callGetAll, callback) {
function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cutoff) {
batchedGetAll(
() => statusesStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
() => statusesStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
results => {
results.forEach(result => {
statusesStore.delete(result.id)
threadsStore.delete(result.id)
let req = statusTimelinesStore.index('statusId').getAllKeys(IDBKeyRange.only(result.id))
req.onsuccess = e => {
let keys = e.target.result
keys.forEach(key => {
statusTimelinesStore.delete(key)
})
}
results.forEach(statusId => {
statusesStore.delete(statusId)
deleteAll(
statusTimelinesStore,
statusTimelinesStore.index('statusId'),
IDBKeyRange.only(statusId)
)
deleteAll(
threadsStore,
threadsStore,
createThreadKeyRange(statusId)
)
})
}
)
@ -53,17 +57,15 @@ function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cut
function cleanupNotifications (notificationsStore, notificationTimelinesStore, cutoff) {
batchedGetAll(
() => notificationsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
() => notificationsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
results => {
results.forEach(result => {
notificationsStore.delete(result.id)
let req = notificationTimelinesStore.index('notificationId').getAllKeys(IDBKeyRange.only(result.id))
req.onsuccess = e => {
let keys = e.target.result
keys.forEach(key => {
notificationTimelinesStore.delete(key)
})
}
results.forEach(notificationId => {
notificationsStore.delete(notificationId)
deleteAll(
notificationTimelinesStore,
notificationTimelinesStore.index('notificationId'),
IDBKeyRange.only(notificationId)
)
})
}
)
@ -71,18 +73,15 @@ function cleanupNotifications (notificationsStore, notificationTimelinesStore, c
function cleanupAccounts (accountsStore, pinnedStatusesStore, cutoff) {
batchedGetAll(
() => accountsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
(results) => {
results.forEach(result => {
accountsStore.delete(result.id)
let keyRange = IDBKeyRange.bound(result.id + '\u0000', result.id + '\u0000\uffff')
let req = pinnedStatusesStore.getAllKeys(keyRange)
req.onsuccess = e => {
let keys = e.target.result
keys.forEach(key => {
pinnedStatusesStore.delete(key)
})
}
() => accountsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
results => {
results.forEach(accountId => {
accountsStore.delete(accountId)
deleteAll(
pinnedStatusesStore,
pinnedStatusesStore,
createPinnedStatusKeyRange(accountId)
)
})
}
)
@ -90,10 +89,10 @@ function cleanupAccounts (accountsStore, pinnedStatusesStore, cutoff) {
function cleanupRelationships (relationshipsStore, cutoff) {
batchedGetAll(
() => relationshipsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
(results) => {
results.forEach(result => {
relationshipsStore.delete(result.id)
() => relationshipsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
results => {
results.forEach(relationshipId => {
relationshipsStore.delete(relationshipId)
})
}
)

18
routes/_database/constants.js
View file

@ -1,12 +1,12 @@
export const STATUSES_STORE = 'statuses-v1'
export const STATUS_TIMELINES_STORE = 'status_timelines-v1'
export const META_STORE = 'meta-v1'
export const ACCOUNTS_STORE = 'accounts-v1'
export const RELATIONSHIPS_STORE = 'relationships-v1'
export const NOTIFICATIONS_STORE = 'notifications-v1'
export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines-v1'
export const PINNED_STATUSES_STORE = 'pinned_statuses-v1'
export const THREADS_STORE = 'threads-v1'
export const STATUSES_STORE = 'statuses-v2'
export const STATUS_TIMELINES_STORE = 'status_timelines-v2'
export const META_STORE = 'meta-v2'
export const ACCOUNTS_STORE = 'accounts-v2'
export const RELATIONSHIPS_STORE = 'relationships-v2'
export const NOTIFICATIONS_STORE = 'notifications-v2'
export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines-v2'
export const PINNED_STATUSES_STORE = 'pinned_statuses-v2'
export const THREADS_STORE = 'threads-v2'
export const TIMESTAMP = '__pinafore_ts'
export const ACCOUNT_ID = '__pinafore_acct_id'

64
routes/_database/databaseLifecycle.js
View file

@ -13,10 +13,12 @@ import {
STATUS_ID
} from './constants'
import forEach from 'lodash/forEach'
const openReqs = {}
const databaseCache = {}
const DB_VERSION = 6
const DB_VERSION = 7
export function getDatabase (instanceName) {
if (!instanceName) {
@ -35,28 +37,46 @@ export function getDatabase (instanceName) {
}
req.onupgradeneeded = (e) => {
let db = req.result
let tx = e.currentTarget.transaction
if (e.oldVersion < 5) {
db.createObjectStore(STATUSES_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(STATUS_TIMELINES_STORE)
.createIndex('statusId', '')
db.createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(NOTIFICATION_TIMELINES_STORE)
.createIndex('notificationId', '')
db.createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(META_STORE)
db.createObjectStore(PINNED_STATUSES_STORE)
tx.objectStore(STATUSES_STORE).createIndex(REBLOG_ID, REBLOG_ID)
tx.objectStore(NOTIFICATIONS_STORE).createIndex('statusId', 'statusId')
db.createObjectStore(THREADS_STORE)
function createObjectStore (name, init, indexes) {
let store = init
? db.createObjectStore(name, init)
: db.createObjectStore(name)
if (indexes) {
forEach(indexes, (indexValue, indexKey) => {
store.createIndex(indexKey, indexValue)
})
}
if (e.oldVersion < 6) {
tx.objectStore(NOTIFICATIONS_STORE).createIndex(STATUS_ID, STATUS_ID)
}
if (e.oldVersion < 7) {
createObjectStore(STATUSES_STORE, {keyPath: 'id'}, {
[TIMESTAMP]: TIMESTAMP,
[REBLOG_ID]: REBLOG_ID
})
createObjectStore(STATUS_TIMELINES_STORE, null, {
'statusId': ''
})
createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'}, {
[TIMESTAMP]: TIMESTAMP,
[STATUS_ID]: STATUS_ID
})
createObjectStore(NOTIFICATION_TIMELINES_STORE, null, {
'notificationId': ''
})
createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'}, {
[TIMESTAMP]: TIMESTAMP
})
createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'}, {
[TIMESTAMP]: TIMESTAMP
})
createObjectStore(THREADS_STORE, null, {
'statusId': ''
})
createObjectStore(PINNED_STATUSES_STORE, null, {
'statusId': ''
})
createObjectStore(META_STORE)
}
}
req.onsuccess = () => resolve(req.result)

47
routes/_database/keys.js Normal file
View file

@ -0,0 +1,47 @@
import { toReversePaddedBigInt, zeroPad } from '../_utils/sorting'
//
// timelines
//
export function createTimelineId (timeline, id) {
// reverse chronological order, prefixed by timeline
return timeline + '\u0000' + toReversePaddedBigInt(id)
}
export function createTimelineKeyRange (timeline, maxId) {
let negBigInt = maxId && toReversePaddedBigInt(maxId)
let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000')
let end = timeline + '\u0000\uffff'
return IDBKeyRange.bound(start, end, true, true)
}
//
// threads
//
export function createThreadId (statusId, i) {
return statusId + '\u0000' + zeroPad(i, 5)
}
export function createThreadKeyRange (statusId) {
return IDBKeyRange.bound(
statusId + '\u0000',
statusId + '\u0000\uffff'
)
}
//
// pinned statues
//
export function createPinnedStatusId (accountId, i) {
return accountId + '\u0000' + zeroPad(i, 3)
}
export function createPinnedStatusKeyRange (accountId) {
return IDBKeyRange.bound(
accountId + '\u0000',
accountId + '\u0000\uffff'
)
}

110
routes/_database/timelines.js
View file

@ -1,4 +1,5 @@
import { toPaddedBigInt, toReversePaddedBigInt } from '../_utils/sorting'
import difference from 'lodash/difference'
import times from 'lodash/times'
import { cloneForStorage } from './helpers'
import { dbPromise, getDatabase } from './databaseLifecycle'
import {
@ -16,13 +17,15 @@ import {
REBLOG_ID,
STATUS_ID, THREADS_STORE
} from './constants'
function createTimelineKeyRange (timeline, maxId) {
let negBigInt = maxId && toReversePaddedBigInt(maxId)
let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000')
let end = timeline + '\u0000\uffff'
return IDBKeyRange.bound(start, end, true, true)
}
import {
createThreadKeyRange,
createTimelineKeyRange,
createTimelineId,
createThreadId,
createPinnedStatusKeyRange,
createPinnedStatusId
} from './keys'
import { deleteAll } from './utils'
function cacheStatus (status, instanceName) {
setInCache(statusesCache, instanceName, status.id, status)
@ -80,7 +83,8 @@ async function getStatusThread (instanceName, statusId) {
const db = await getDatabase(instanceName)
return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
let [ threadsStore, statusesStore, accountsStore ] = stores
threadsStore.get(statusId).onsuccess = e => {
let keyRange = createThreadKeyRange(statusId)
threadsStore.getAll(keyRange).onsuccess = e => {
let thread = e.target.result
let res = new Array(thread.length)
thread.forEach((otherStatusId, i) => {
@ -177,11 +181,6 @@ function fetchNotification (notificationsStore, statusesStore, accountsStore, id
}
}
function createTimelineId (timeline, id) {
// reverse chronological order, prefixed by timeline
return timeline + '\u0000' + toReversePaddedBigInt(id)
}
async function insertTimelineNotifications (instanceName, timeline, notifications) {
for (let notification of notifications) {
setInCache(notificationsCache, instanceName, notification.id, notification)
@ -224,10 +223,18 @@ async function insertStatusThread (instanceName, statusId, statuses) {
let storeNames = [THREADS_STORE, STATUSES_STORE, ACCOUNTS_STORE]
await dbPromise(db, storeNames, 'readwrite', (stores) => {
let [ threadsStore, statusesStore, accountsStore ] = stores
threadsStore.put(statuses.map(_ => _.id), statusId)
for (let status of statuses) {
storeStatus(statusesStore, accountsStore, status)
threadsStore.getAllKeys(createThreadKeyRange(statusId)).onsuccess = e => {
let existingKeys = e.target.result
let newKeys = times(statuses.length, i => createThreadId(statusId, i))
let keysToDelete = difference(existingKeys, newKeys)
for (let key of keysToDelete) {
threadsStore.delete(key)
}
}
statuses.forEach((otherStatus, i) => {
storeStatus(statusesStore, accountsStore, otherStatus)
threadsStore.put(otherStatus.id, createThreadId(statusId, i))
})
})
}
@ -323,7 +330,8 @@ export async function deleteStatusesAndNotifications (instanceName, statusIds, n
STATUS_TIMELINES_STORE,
NOTIFICATIONS_STORE,
NOTIFICATION_TIMELINES_STORE,
PINNED_STATUSES_STORE
PINNED_STATUSES_STORE,
THREADS_STORE
]
await dbPromise(db, storeNames, 'readwrite', (stores) => {
let [
@ -331,33 +339,41 @@ export async function deleteStatusesAndNotifications (instanceName, statusIds, n
statusTimelinesStore,
notificationsStore,
notificationTimelinesStore,
pinnedStatusesStore
pinnedStatusesStore,
threadsStore
] = stores
function deleteStatus (statusId) {
pinnedStatusesStore.delete(statusId).onerror = e => {
e.preventDefault()
e.stopPropagation()
}
statusesStore.delete(statusId)
let getAllReq = statusTimelinesStore.index('statusId')
.getAllKeys(IDBKeyRange.only(statusId))
getAllReq.onsuccess = e => {
for (let result of e.target.result) {
statusTimelinesStore.delete(result)
}
}
deleteAll(
pinnedStatusesStore,
pinnedStatusesStore.index('statusId'),
IDBKeyRange.only(statusId)
)
deleteAll(
statusTimelinesStore,
statusTimelinesStore.index('statusId'),
IDBKeyRange.only(statusId)
)
deleteAll(
threadsStore,
threadsStore.index('statusId'),
IDBKeyRange.only(statusId)
)
deleteAll(
threadsStore,
threadsStore,
createThreadKeyRange(statusId)
)
}
function deleteNotification (notificationId) {
notificationsStore.delete(notificationId)
let getAllReq = notificationTimelinesStore.index('statusId')
.getAllKeys(IDBKeyRange.only(notificationId))
getAllReq.onsuccess = e => {
for (let result of e.target.result) {
notificationTimelinesStore.delete(result)
}
}
deleteAll(
notificationTimelinesStore,
notificationTimelinesStore.index('statusId'),
IDBKeyRange.only(notificationId)
)
}
for (let statusId of statusIds) {
@ -383,7 +399,7 @@ export async function insertPinnedStatuses (instanceName, accountId, statuses) {
let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores
statuses.forEach((status, i) => {
storeStatus(statusesStore, accountsStore, status)
pinnedStatusesStore.put(status.id, accountId + '\u0000' + toPaddedBigInt(i))
pinnedStatusesStore.put(status.id, createPinnedStatusId(accountId, i))
})
})
}
@ -393,10 +409,7 @@ export async function getPinnedStatuses (instanceName, accountId) {
const db = await getDatabase(instanceName)
return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores
let keyRange = IDBKeyRange.bound(
accountId + '\u0000',
accountId + '\u0000\uffff'
)
let keyRange = createPinnedStatusKeyRange(accountId)
pinnedStatusesStore.getAll(keyRange).onsuccess = e => {
let pinnedResults = e.target.result
let res = new Array(pinnedResults.length)
@ -410,19 +423,6 @@ export async function getPinnedStatuses (instanceName, accountId) {
})
}
//
// notifications by status
//
export async function getNotificationIdsForStatus (instanceName, statusId) {
const db = await getDatabase(instanceName)
return dbPromise(db, NOTIFICATIONS_STORE, 'readonly', (notificationStore, callback) => {
notificationStore.index(statusId).getAllKeys(IDBKeyRange.only(statusId)).onsuccess = e => {
callback(Array.from(e.target.result))
}
})
}
//
// update statuses
//

7
routes/_database/utils.js Normal file
View file

@ -0,0 +1,7 @@
export function deleteAll (store, index, keyRange) {
index.getAllKeys(keyRange).onsuccess = e => {
for (let result of e.target.result) {
store.delete(result)
}
}
}

6
routes/_utils/sorting.js
View file

@ -1,7 +1,11 @@
import padStart from 'lodash/padStart'
export function zeroPad (str, toSize) {
return padStart(str, toSize, '0')
}
export function toPaddedBigInt (id) {
return padStart(id, 30, '0')
return zeroPad(id, 30)
}
export function toReversePaddedBigInt (id) {

5
tests/serverActions.js
View file

@ -20,6 +20,11 @@ export async function postAsAdmin (text) {
null, null, false, null, 'public')
}
export async function postReplyAsAdmin (text, inReplyTo) {
return postStatus(instanceName, users.admin.accessToken, text,
inReplyTo, null, false, null, 'public')
}
export async function deleteAsAdmin (statusId) {
return deleteStatus(instanceName, users.admin.accessToken, statusId)
}

44
tests/spec/105-deletes.js
View file

@ -1,6 +1,9 @@
import { foobarRole } from '../roles'
import { getNthStatus } from '../utils'
import { deleteAsAdmin, postAsAdmin } from '../serverActions'
import {
clickToNotificationsAndBackHome, forceOffline, forceOnline, getNthStatus, getUrl, homeNavButton,
sleep
} from '../utils'
import { deleteAsAdmin, postAsAdmin, postReplyAsAdmin } from '../serverActions'
fixture`105-deletes.js`
.page`http://localhost:4002`
@ -9,7 +12,44 @@ test('deleted statuses are removed from the timeline', async t => {
await t.useRole(foobarRole)
.hover(getNthStatus(0))
let status = await postAsAdmin("I'm gonna delete this")
await sleep(1000)
await t.expect(getNthStatus(0).innerText).contains("I'm gonna delete this")
await deleteAsAdmin(status.id)
await sleep(1000)
await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
await clickToNotificationsAndBackHome(t)
await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
await t.navigateTo('/notifications')
await forceOffline()
await t.click(homeNavButton)
await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
await forceOnline()
await t
.navigateTo('/')
.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this")
})
test('deleted statuses are removed from threads', async t => {
await t.useRole(foobarRole)
.hover(getNthStatus(0))
let status = await postAsAdmin("I won't delete this")
let reply = await postReplyAsAdmin('But I will delete this', status.id)
await sleep(5000)
await t.expect(getNthStatus(0).innerText).contains('But I will delete this')
.expect(getNthStatus(1).innerText).contains("I won't delete this")
.click(getNthStatus(1))
.expect(getUrl()).contains('/statuses')
.expect(getNthStatus(0).innerText).contains("I won't delete this")
.expect(getNthStatus(1).innerText).contains('But I will delete this')
await deleteAsAdmin(reply.id)
await sleep(1000)
await t.expect(getNthStatus(1).exists).notOk()
.expect(getNthStatus(0).innerText).contains("I won't delete this")
await t.navigateTo('/')
await forceOffline()
await t.click(getNthStatus(0))
.expect(getUrl()).contains('/statuses')
.expect(getNthStatus(1).exists).notOk()
.expect(getNthStatus(0).innerText).contains("I won't delete this")
await forceOnline()
})

7
tests/utils.js
View file

@ -186,3 +186,10 @@ export async function scrollToStatus (t, n) {
}
await t.hover(getNthStatus(n))
}
export async function clickToNotificationsAndBackHome (t) {
await t.click(notificationsNavButton)
.expect(getUrl()).eql('http://localhost:4002/notifications')
.click(homeNavButton)
.expect(getUrl()).eql('http://localhost:4002/')
}