refactor db usage

This commit is contained in:
Nolan Lawson 2018-01-19 09:18:14 -08:00
parent d659578338
commit a05400b06f
6 changed files with 141 additions and 92 deletions

View file

@ -15,7 +15,7 @@
import StatusListItem from './StatusListItem.html' import StatusListItem from './StatusListItem.html'
import VirtualList from './VirtualList.html' import VirtualList from './VirtualList.html'
import { splice, push } from 'svelte-extras' import { splice, push } from 'svelte-extras'
import worker from 'workerize-loader!../_utils/database/statuses' import worker from 'workerize-loader!../_utils/database/database'
import { mergeStatuses } from '../_utils/statuses' import { mergeStatuses } from '../_utils/statuses'
import { mark, stop } from '../_utils/marks' import { mark, stop } from '../_utils/marks'
import { timelines } from '../_static/timelines' import { timelines } from '../_static/timelines'

View file

@ -0,0 +1,42 @@
import keyval from "idb-keyval"
import debounce from 'lodash/debounce'
import { OBJECT_STORE, getDatabase } from './shared'
const MAX_NUM_STORED_STATUSES = 1000
const CLEANUP_INTERVAL = 60000
async function cleanup(instanceName, timeline) {
const db = await getDatabase(instanceName, timeline)
return await new Promise((resolve, reject) => {
const tx = db.transaction(OBJECT_STORE, 'readwrite')
const store = tx.objectStore(OBJECT_STORE)
const index = store.index('pinafore_id_as_negative_big_int')
store.count().onsuccess = (e) => {
let count = e.target.result
let openKeyCursor = index.openKeyCursor || index.openCursor
openKeyCursor.call(index, null, 'prev').onsuccess = (e) => {
let cursor = e.target.result
if (--count < MAX_NUM_STORED_STATUSES || !cursor) {
return
}
store.delete(cursor.primaryKey).onsuccess = () => {
cursor.continue()
}
}
}
tx.oncomplete = () => resolve()
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
})
}
export const cleanupOldStatuses = debounce(async () => {
console.log('cleanupOldStatuses')
let knownDbs = (await keyval.get('known_dbs')) || {}
let dbNames = Object.keys(knownDbs)
for (let dbName of dbNames) {
let [ instanceName, timeline ] = knownDbs[dbName]
await cleanup(instanceName, timeline)
}
console.log('done cleanupOldStatuses')
}, CLEANUP_INTERVAL)

View file

@ -0,0 +1,36 @@
import { cleanupOldStatuses } from './cleanup'
import { OBJECT_STORE, getDatabase, doTransaction } from './shared'
import { toReversePaddedBigInt, transformStatusForStorage } from './utils'
export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
const db = await getDatabase(instanceName, timeline)
return await new Promise((resolve, reject) => {
const tx = db.transaction(OBJECT_STORE, 'readonly')
const store = tx.objectStore(OBJECT_STORE)
const index = store.index('pinafore_id_as_negative_big_int')
let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
let res
index.getAll(query, limit).onsuccess = (e) => {
res = e.target.result
}
tx.oncomplete = () => resolve(res)
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
})
}
export async function insertStatuses(instanceName, timeline, statuses) {
cleanupOldStatuses()
const db = await getDatabase(instanceName, timeline)
return await new Promise((resolve, reject) => {
const tx = db.transaction(OBJECT_STORE, 'readwrite')
const store = tx.objectStore(OBJECT_STORE)
for (let status of statuses) {
store.put(transformStatusForStorage(status))
}
tx.oncomplete = () => resolve()
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
})
}

View file

@ -0,0 +1,40 @@
import keyval from "idb-keyval"
const databaseCache = {}
export const OBJECT_STORE = 'statuses'
export function createDbName(instanceName, timeline) {
return `${OBJECT_STORE}_${instanceName}_${timeline}`
}
export function getDatabase(instanceName, timeline) {
const key = `${instanceName}_${timeline}`
if (databaseCache[key]) {
return Promise.resolve(databaseCache[key])
}
let dbName = createDbName(instanceName, timeline)
keyval.get('known_dbs').then(knownDbs => {
knownDbs = knownDbs || {}
knownDbs[dbName] = [instanceName, timeline]
keyval.set('known_dbs', knownDbs)
})
databaseCache[key] = new Promise((resolve, reject) => {
let req = indexedDB.open(dbName, 1)
req.onerror = reject
req.onblocked = () => {
console.log('idb blocked')
}
req.onupgradeneeded = () => {
let db = req.result;
let oStore = db.createObjectStore(OBJECT_STORE, {
keyPath: 'id'
})
oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
}
req.onsuccess = () => resolve(req.result)
})
return databaseCache[key]
}

View file

@ -1,91 +0,0 @@
import keyval from 'idb-keyval'
import cloneDeep from 'lodash/cloneDeep'
import padStart from 'lodash/padStart'
const STORE = 'statuses'
const databaseCache = {}
function toPaddedBigInt(id) {
return padStart(id, 30, '0')
}
function toReversePaddedBigInt(id) {
let bigInt = toPaddedBigInt(id)
let res = ''
for (let i = 0; i < bigInt.length; i++) {
res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
}
return res
}
function transformStatusForStorage(status) {
status = cloneDeep(status)
status.pinafore_id_as_big_int = toPaddedBigInt(status.id)
status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
status.pinafore_stale = true
return status
}
function getDatabase(instanceName, timeline) {
const key = `${instanceName}_${timeline}`
if (databaseCache[key]) {
return Promise.resolve(databaseCache[key])
}
let objectStoreName = `${STORE}_${key}`
keyval.get('known_dbs').then(knownDbs => {
knownDbs = knownDbs || {}
knownDbs[objectStoreName] = true
keyval.set('known_dbs', knownDbs)
})
databaseCache[key] = new Promise((resolve, reject) => {
let req = indexedDB.open(objectStoreName, 1)
req.onerror = reject
req.onblocked = () => {
console.log('idb blocked')
}
req.onupgradeneeded = () => {
let db = req.result;
let oStore = db.createObjectStore(STORE, {
keyPath: 'id'
})
oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
}
req.onsuccess = () => resolve(req.result)
})
return databaseCache[key]
}
export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
const db = await getDatabase(instanceName, timeline)
return await new Promise((resolve, reject) => {
const tx = db.transaction(STORE, 'readonly')
const store = tx.objectStore(STORE)
const index = store.index('pinafore_id_as_negative_big_int')
let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
let res
index.getAll(query, limit).onsuccess = (e) => {
res = e.target.result
}
tx.oncomplete = () => resolve(res)
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
})
}
export async function insertStatuses(instanceName, timeline, statuses) {
const db = await getDatabase(instanceName, timeline)
return await new Promise((resolve, reject) => {
const tx = db.transaction(STORE, 'readwrite')
const store = tx.objectStore(STORE)
for (let status of statuses) {
store.put(transformStatusForStorage(status))
}
tx.oncomplete = () => resolve()
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
})
}

View file

@ -0,0 +1,22 @@
import cloneDeep from 'lodash/cloneDeep'
import padStart from 'lodash/padStart'
export function toPaddedBigInt (id) {
return padStart(id, 30, '0')
}
export function toReversePaddedBigInt (id) {
let bigInt = toPaddedBigInt(id)
let res = ''
for (let i = 0; i < bigInt.length; i++) {
res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
}
return res
}
export function transformStatusForStorage (status) {
status = cloneDeep(status)
status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
status.pinafore_stale = true
return status
}