data.coop/pinafore
8
0
Fork 0
Code Issues 1 Pull requests Packages Projects Releases Wiki Activity

simplify idb schema

Browse source
This commit is contained in:
Nolan Lawson 2018-03-08 23:18:18 -08:00
parent 443701605f
commit f42a07162b
5 changed files with 57 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
routes/_database/cleanup.js
View file

@ -4,6 +4,7 @@ import {
ACCOUNTS_STORE, ACCOUNTS_STORE,
NOTIFICATION_TIMELINES_STORE, NOTIFICATION_TIMELINES_STORE,
NOTIFICATIONS_STORE, NOTIFICATIONS_STORE,
PINNED_STATUSES_STORE,
RELATIONSHIPS_STORE, RELATIONSHIPS_STORE,
STATUS_TIMELINES_STORE, STATUS_TIMELINES_STORE,
STATUSES_STORE, STATUSES_STORE,
@ -38,11 +39,11 @@ function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cut
results.forEach(result => { results.forEach(result => {
statusesStore.delete(result.id) statusesStore.delete(result.id)
threadsStore.delete(result.id) threadsStore.delete(result.id)
let req = statusTimelinesStore.index('statusId').getAll(IDBKeyRange.only(result.id)) let req = statusTimelinesStore.index('statusId').getAllKeys(IDBKeyRange.only(result.id))
req.onsuccess = e => { req.onsuccess = e => {
let results = e.target.result let keys = e.target.result
results.forEach(result => { keys.forEach(key => {
statusTimelinesStore.delete(result.id) statusTimelinesStore.delete(key)
}) })
} }
}) })
@ -56,11 +57,11 @@ function cleanupNotifications (notificationsStore, notificationTimelinesStore, c
results => { results => {
results.forEach(result => { results.forEach(result => {
notificationsStore.delete(result.id) notificationsStore.delete(result.id)
let req = notificationTimelinesStore.index('notificationId').getAll(IDBKeyRange.only(result.id)) let req = notificationTimelinesStore.index('notificationId').getAllKeys(IDBKeyRange.only(result.id))
req.onsuccess = e => { req.onsuccess = e => {
let results = e.target.result let keys = e.target.result
results.forEach(result => { keys.forEach(key => {
notificationTimelinesStore.delete(result.id) notificationTimelinesStore.delete(key)
}) })
} }
}) })
@ -68,12 +69,20 @@ function cleanupNotifications (notificationsStore, notificationTimelinesStore, c
) )
} }
function cleanupAccounts (accountsStore, cutoff) { function cleanupAccounts (accountsStore, pinnedStatusesStore, cutoff) {
batchedGetAll( batchedGetAll(
() => accountsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), () => accountsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE),
(results) => { (results) => {
results.forEach(result => { results.forEach(result => {
accountsStore.delete(result.id) accountsStore.delete(result.id)
let keyRange = IDBKeyRange.bound(result.id + '\u0000', result.id + '\u0000\uffff')
let req = pinnedStatusesStore.getAllKeys(keyRange)
req.onsuccess = e => {
let keys = e.target.result
keys.forEach(key => {
pinnedStatusesStore.delete(key)
})
}
}) })
} }
) )
@ -101,7 +110,8 @@ async function cleanup (instanceName) {
NOTIFICATION_TIMELINES_STORE, NOTIFICATION_TIMELINES_STORE,
ACCOUNTS_STORE, ACCOUNTS_STORE,
RELATIONSHIPS_STORE, RELATIONSHIPS_STORE,
THREADS_STORE THREADS_STORE,
PINNED_STATUSES_STORE
] ]
await dbPromise(db, storeNames, 'readwrite', (stores) => { await dbPromise(db, storeNames, 'readwrite', (stores) => {
let [ let [
@ -111,14 +121,15 @@ async function cleanup (instanceName) {
notificationTimelinesStore, notificationTimelinesStore,
accountsStore, accountsStore,
relationshipsStore, relationshipsStore,
threadsStore threadsStore,
pinnedStatusesStore
] = stores ] = stores
let cutoff = Date.now() - TIME_AGO let cutoff = Date.now() - TIME_AGO
cleanupStatuses(statusesStore, statusTimelinesStore, threadsStore, cutoff) cleanupStatuses(statusesStore, statusTimelinesStore, threadsStore, cutoff)
cleanupNotifications(notificationsStore, notificationTimelinesStore, cutoff) cleanupNotifications(notificationsStore, notificationTimelinesStore, cutoff)
cleanupAccounts(accountsStore, cutoff) cleanupAccounts(accountsStore, pinnedStatusesStore, cutoff)
cleanupRelationships(relationshipsStore, cutoff) cleanupRelationships(relationshipsStore, cutoff)
}) })
stop(`cleanup:${instanceName}`) stop(`cleanup:${instanceName}`)

18
routes/_database/constants.js
View file

@ -1,12 +1,12 @@
export const STATUSES_STORE = 'statuses' export const STATUSES_STORE = 'statuses-v1'
export const STATUS_TIMELINES_STORE = 'status_timelines' export const STATUS_TIMELINES_STORE = 'status_timelines-v1'
export const META_STORE = 'meta' export const META_STORE = 'meta-v1'
export const ACCOUNTS_STORE = 'accounts' export const ACCOUNTS_STORE = 'accounts-v1'
export const RELATIONSHIPS_STORE = 'relationships' export const RELATIONSHIPS_STORE = 'relationships-v1'
export const NOTIFICATIONS_STORE = 'notifications' export const NOTIFICATIONS_STORE = 'notifications-v1'
export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines' export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines-v1'
export const PINNED_STATUSES_STORE = 'pinned_statuses' export const PINNED_STATUSES_STORE = 'pinned_statuses-v1'
export const THREADS_STORE = 'threads' export const THREADS_STORE = 'threads-v1'
export const TIMESTAMP = '__pinafore_ts' export const TIMESTAMP = '__pinafore_ts'
export const ACCOUNT_ID = '__pinafore_acct_id' export const ACCOUNT_ID = '__pinafore_acct_id'

28
routes/_database/databaseLifecycle.js
View file

@ -7,13 +7,15 @@ import {
NOTIFICATIONS_STORE, NOTIFICATIONS_STORE,
NOTIFICATION_TIMELINES_STORE, NOTIFICATION_TIMELINES_STORE,
PINNED_STATUSES_STORE, PINNED_STATUSES_STORE,
TIMESTAMP, REBLOG_ID, THREADS_STORE TIMESTAMP,
REBLOG_ID,
THREADS_STORE
} from './constants' } from './constants'
const openReqs = {} const openReqs = {}
const databaseCache = {} const databaseCache = {}
const DB_VERSION = 4 const DB_VERSION = 5
export function getDatabase (instanceName) { export function getDatabase (instanceName) {
if (!instanceName) { if (!instanceName) {
@ -33,30 +35,24 @@ export function getDatabase (instanceName) {
req.onupgradeneeded = (e) => { req.onupgradeneeded = (e) => {
let db = req.result let db = req.result
let tx = e.currentTarget.transaction let tx = e.currentTarget.transaction
if (e.oldVersion < 1) { if (e.oldVersion < 5) {
db.createObjectStore(STATUSES_STORE, {keyPath: 'id'}) db.createObjectStore(STATUSES_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP) .createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(STATUS_TIMELINES_STORE, {keyPath: 'id'}) db.createObjectStore(STATUS_TIMELINES_STORE)
.createIndex('statusId', 'statusId') .createIndex('statusId', '')
db.createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'}) db.createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP) .createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(NOTIFICATION_TIMELINES_STORE, {keyPath: 'id'}) db.createObjectStore(NOTIFICATION_TIMELINES_STORE)
.createIndex('notificationId', 'notificationId') .createIndex('notificationId', '')
db.createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'}) db.createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP) .createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'}) db.createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'})
.createIndex(TIMESTAMP, TIMESTAMP) .createIndex(TIMESTAMP, TIMESTAMP)
db.createObjectStore(META_STORE, {keyPath: 'key'}) db.createObjectStore(META_STORE)
db.createObjectStore(PINNED_STATUSES_STORE, {keyPath: 'id'}) db.createObjectStore(PINNED_STATUSES_STORE)
}
if (e.oldVersion < 2) {
tx.objectStore(STATUSES_STORE).createIndex(REBLOG_ID, REBLOG_ID) tx.objectStore(STATUSES_STORE).createIndex(REBLOG_ID, REBLOG_ID)
}
if (e.oldVersion < 3) {
tx.objectStore(NOTIFICATIONS_STORE).createIndex('statusId', 'statusId') tx.objectStore(NOTIFICATIONS_STORE).createIndex('statusId', 'statusId')
} db.createObjectStore(THREADS_STORE)
if (e.oldVersion < 4) {
db.createObjectStore(THREADS_STORE, {keyPath: 'id'})
} }
} }
req.onsuccess = () => resolve(req.result) req.onsuccess = () => resolve(req.result)

7
routes/_database/meta.js
View file

@ -9,7 +9,7 @@ async function getMetaProperty (instanceName, key) {
const db = await getDatabase(instanceName) const db = await getDatabase(instanceName)
let result = await dbPromise(db, META_STORE, 'readonly', (store, callback) => { let result = await dbPromise(db, META_STORE, 'readonly', (store, callback) => {
store.get(key).onsuccess = (e) => { store.get(key).onsuccess = (e) => {
callback(e.target.result && e.target.result.value) callback(e.target.result)
} }
}) })
setInCache(metaCache, instanceName, key, result) setInCache(metaCache, instanceName, key, result)
@ -20,10 +20,7 @@ async function setMetaProperty (instanceName, key, value) {
setInCache(metaCache, instanceName, key, value) setInCache(metaCache, instanceName, key, value)
const db = await getDatabase(instanceName) const db = await getDatabase(instanceName)
return dbPromise(db, META_STORE, 'readwrite', (store) => { return dbPromise(db, META_STORE, 'readwrite', (store) => {
store.put({ store.put(value, key)
key: key,
value: value
})
}) })
} }

34
routes/_database/timelines.js
View file

@ -46,8 +46,8 @@ async function getNotificationTimeline (instanceName, timeline, maxId, limit) {
timelineStore.getAll(keyRange, limit).onsuccess = e => { timelineStore.getAll(keyRange, limit).onsuccess = e => {
let timelineResults = e.target.result let timelineResults = e.target.result
let res = new Array(timelineResults.length) let res = new Array(timelineResults.length)
timelineResults.forEach((timelineResult, i) => { timelineResults.forEach((notificationId, i) => {
fetchNotification(notificationsStore, statusesStore, accountsStore, timelineResult.notificationId, notification => { fetchNotification(notificationsStore, statusesStore, accountsStore, notificationId, notification => {
res[i] = notification res[i] = notification
}) })
}) })
@ -65,8 +65,8 @@ async function getStatusTimeline (instanceName, timeline, maxId, limit) {
getReq.onsuccess = e => { getReq.onsuccess = e => {
let timelineResults = e.target.result let timelineResults = e.target.result
let res = new Array(timelineResults.length) let res = new Array(timelineResults.length)
timelineResults.forEach((timelineResult, i) => { timelineResults.forEach((statusId, i) => {
fetchStatus(statusesStore, accountsStore, timelineResult.statusId, status => { fetchStatus(statusesStore, accountsStore, statusId, status => {
res[i] = status res[i] = status
}) })
}) })
@ -81,7 +81,7 @@ async function getStatusThread (instanceName, statusId) {
return dbPromise(db, storeNames, 'readonly', (stores, callback) => { return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
let [ threadsStore, statusesStore, accountsStore ] = stores let [ threadsStore, statusesStore, accountsStore ] = stores
threadsStore.get(statusId).onsuccess = e => { threadsStore.get(statusId).onsuccess = e => {
let thread = e.target.result.thread let thread = e.target.result
let res = new Array(thread.length) let res = new Array(thread.length)
thread.forEach((otherStatusId, i) => { thread.forEach((otherStatusId, i) => {
fetchStatus(statusesStore, accountsStore, otherStatusId, status => { fetchStatus(statusesStore, accountsStore, otherStatusId, status => {
@ -196,10 +196,7 @@ async function insertTimelineNotifications (instanceName, timeline, notification
let [ timelineStore, notificationsStore, accountsStore, statusesStore ] = stores let [ timelineStore, notificationsStore, accountsStore, statusesStore ] = stores
for (let notification of notifications) { for (let notification of notifications) {
storeNotification(notificationsStore, statusesStore, accountsStore, notification) storeNotification(notificationsStore, statusesStore, accountsStore, notification)
timelineStore.put({ timelineStore.put(notification.id, createTimelineId(timeline, notification.id))
id: createTimelineId(timeline, notification.id),
notificationId: notification.id
})
} }
}) })
} }
@ -214,10 +211,7 @@ async function insertTimelineStatuses (instanceName, timeline, statuses) {
let [ timelineStore, statusesStore, accountsStore ] = stores let [ timelineStore, statusesStore, accountsStore ] = stores
for (let status of statuses) { for (let status of statuses) {
storeStatus(statusesStore, accountsStore, status) storeStatus(statusesStore, accountsStore, status)
timelineStore.put({ timelineStore.put(status.id, createTimelineId(timeline, status.id))
id: createTimelineId(timeline, status.id),
statusId: status.id
})
} }
}) })
} }
@ -230,10 +224,7 @@ async function insertStatusThread (instanceName, statusId, statuses) {
let storeNames = [THREADS_STORE, STATUSES_STORE, ACCOUNTS_STORE] let storeNames = [THREADS_STORE, STATUSES_STORE, ACCOUNTS_STORE]
await dbPromise(db, storeNames, 'readwrite', (stores) => { await dbPromise(db, storeNames, 'readwrite', (stores) => {
let [ threadsStore, statusesStore, accountsStore ] = stores let [ threadsStore, statusesStore, accountsStore ] = stores
threadsStore.put({ threadsStore.put(statuses.map(_ => _.id), statusId)
id: statusId,
thread: statuses.map(_ => _.id)
})
for (let status of statuses) { for (let status of statuses) {
storeStatus(statusesStore, accountsStore, status) storeStatus(statusesStore, accountsStore, status)
} }
@ -374,10 +365,7 @@ export async function insertPinnedStatuses (instanceName, accountId, statuses) {
let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores
statuses.forEach((status, i) => { statuses.forEach((status, i) => {
storeStatus(statusesStore, accountsStore, status) storeStatus(statusesStore, accountsStore, status)
pinnedStatusesStore.put({ pinnedStatusesStore.put(status.id, accountId + '\u0000' + toPaddedBigInt(i))
id: accountId + '\u0000' + toPaddedBigInt(i),
statusId: status.id
})
}) })
}) })
} }
@ -394,8 +382,8 @@ export async function getPinnedStatuses (instanceName, accountId) {
pinnedStatusesStore.getAll(keyRange).onsuccess = e => { pinnedStatusesStore.getAll(keyRange).onsuccess = e => {
let pinnedResults = e.target.result let pinnedResults = e.target.result
let res = new Array(pinnedResults.length) let res = new Array(pinnedResults.length)
pinnedResults.forEach((pinnedResult, i) => { pinnedResults.forEach((statusId, i) => {
fetchStatus(statusesStore, accountsStore, pinnedResult.statusId, status => { fetchStatus(statusesStore, accountsStore, statusId, status => {
res[i] = status res[i] = status
}) })
}) })