data.coop/pinafore
8
0
Fork 0
Code Issues 1 Pull requests Packages Projects Releases Wiki Activity

fix: fix pleroma snowflake IDs for real (#1084)

Browse source 
fixes #1082
This commit is contained in:
Nolan Lawson 2019-03-05 20:25:10 -08:00 committed by GitHub
parent 8665a18ac0
commit c0918ccdc3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 173 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/routes/_database/constants.js
View file

@ -16,8 +16,8 @@ export const USERNAME_LOWERCASE = '__pinafore_acct_lc'
export const DB_VERSION_INITIAL = 9
export const DB_VERSION_SEARCH_ACCOUNTS = 10
export const DB_VERSION_SNOWFLAKE_IDS = 11
export const DB_VERSION_SNOWFLAKE_IDS = 12 // 11 skipped because of mistake deployed to dev.pinafore.social
// Using an object for these so that unit tests can change them
export const DB_VERSION_CURRENT = { version: 11 }
export const DB_VERSION_CURRENT = { version: 12 }
export const CURRENT_TIME = { now: () => Date.now() }

18
src/routes/_utils/statusIdSorting.js
View file

@ -4,13 +4,12 @@
import { padStart } from './lodash-lite'
// Unfortunately base62 ordering is not the same as JavaScript's default ASCII ordering,
// used both for JS string comparisons as well as IndexedDB ordering.
const BASE62_ALPHABET = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
// Pleroma uses the 0-9A-Za-z alphabet for base62, which is the same as ASCII, which
// is the same as JavaScript sort order and IndexedDB order.
const MIN_CHAR_CODE = 48 // '0'.charCodeAt(0)
const MAX_CHAR_CODE = 122 // 'z'.charCodeAt(0)
const MAX_ID_LENGTH = 30 // assume that Mastodon/Pleroma IDs won't get any bigger than this
const BASE62_LOOKUP = new Map(BASE62_ALPHABET.split('').map((char, i) => ([char, i])))
export function zeroPad (str, toSize) {
return padStart(str, toSize, '0')
}
@ -20,13 +19,12 @@ export function toPaddedBigInt (id) {
}
export function toReversePaddedBigInt (id) {
let padded = zeroPad(id, MAX_ID_LENGTH)
let padded = toPaddedBigInt(id)
let reversed = ''
for (let i = 0; i < padded.length; i++) {
let char = padded.charAt(i)
let idx = BASE62_LOOKUP.get(char)
let reverseIdx = BASE62_ALPHABET.length - 1 - idx
reversed += BASE62_ALPHABET[reverseIdx]
let charCode = padded.charCodeAt(i)
let inverseCharCode = MIN_CHAR_CODE + MAX_CHAR_CODE - charCode
reversed += String.fromCharCode(inverseCharCode)
}
return reversed
}

2
tests/unit/test-database.js
View file

@ -153,7 +153,7 @@ describe('test-database.js', function () {
await deleteDatabase(INSTANCE_NAME)
})
it('migrates from v10 to v11', async () => {
it('migrates to snowflake IDs', async () => {
// open the db using the old version
DB_VERSION_CURRENT.version = DB_VERSION_SEARCH_ACCOUNTS
await getDatabase(INSTANCE_NAME)

163
tests/unit/test-id-sorting.js
View file

@ -13,6 +13,15 @@ function gt (a, b) {
}
describe('test-id-sorting.js', () => {
it('basic id sorting', () => {
assert.deepStrictEqual(toPaddedBigInt('0'), '000000000000000000000000000000')
assert.deepStrictEqual(toPaddedBigInt('1'), '000000000000000000000000000001')
assert.deepStrictEqual(toPaddedBigInt('z'), '00000000000000000000000000000z')
assert.deepStrictEqual(toReversePaddedBigInt('0'), 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz')
assert.deepStrictEqual(toReversePaddedBigInt('1'), 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzy')
assert.deepStrictEqual(toReversePaddedBigInt('z'), 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzz0')
})
it('can sort mastodon IDs correctly', () => {
let id1 = '1'
let id2 = '2'
@ -94,7 +103,75 @@ describe('test-id-sorting.js', () => {
assert.deepStrictEqual(toReversePaddedBigInt(id5), toReversePaddedBigInt(id5))
})
it('can sort pleroma ids - more examples', () => {
it('can sort base62 IDs correctly 2', () => {
let id1 = '0'
let id2 = 'A'
let id3 = 'T'
let id4 = 'a'
let id5 = 'z'
lt(toPaddedBigInt(id1), toPaddedBigInt(id2))
lt(toPaddedBigInt(id2), toPaddedBigInt(id3))
lt(toPaddedBigInt(id3), toPaddedBigInt(id4))
lt(toPaddedBigInt(id4), toPaddedBigInt(id5))
lt(toPaddedBigInt(id1), toPaddedBigInt(id5))
lt(toPaddedBigInt(id2), toPaddedBigInt(id5))
lt(toPaddedBigInt(id3), toPaddedBigInt(id5))
lt(toPaddedBigInt(id2), toPaddedBigInt(id4))
assert.deepStrictEqual(toPaddedBigInt(id1), toPaddedBigInt(id1))
assert.deepStrictEqual(toPaddedBigInt(id2), toPaddedBigInt(id2))
assert.deepStrictEqual(toPaddedBigInt(id3), toPaddedBigInt(id3))
assert.deepStrictEqual(toPaddedBigInt(id4), toPaddedBigInt(id4))
assert.deepStrictEqual(toPaddedBigInt(id5), toPaddedBigInt(id5))
gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id2))
gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id3))
gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id4))
gt(toReversePaddedBigInt(id4), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id4))
assert.deepStrictEqual(toReversePaddedBigInt(id1), toReversePaddedBigInt(id1))
assert.deepStrictEqual(toReversePaddedBigInt(id2), toReversePaddedBigInt(id2))
assert.deepStrictEqual(toReversePaddedBigInt(id3), toReversePaddedBigInt(id3))
assert.deepStrictEqual(toReversePaddedBigInt(id4), toReversePaddedBigInt(id4))
assert.deepStrictEqual(toReversePaddedBigInt(id5), toReversePaddedBigInt(id5))
})
it('can sort base62 IDs correctly 3', () => {
let id1 = 'a'
let id2 = 'z'
let id3 = 'a0'
let id4 = 'xx0'
let id5 = 'a000'
lt(toPaddedBigInt(id1), toPaddedBigInt(id2))
lt(toPaddedBigInt(id2), toPaddedBigInt(id3))
lt(toPaddedBigInt(id3), toPaddedBigInt(id4))
lt(toPaddedBigInt(id4), toPaddedBigInt(id5))
lt(toPaddedBigInt(id1), toPaddedBigInt(id5))
lt(toPaddedBigInt(id2), toPaddedBigInt(id5))
lt(toPaddedBigInt(id3), toPaddedBigInt(id5))
lt(toPaddedBigInt(id2), toPaddedBigInt(id4))
gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id2))
gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id3))
gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id4))
gt(toReversePaddedBigInt(id4), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id1), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id3), toReversePaddedBigInt(id5))
gt(toReversePaddedBigInt(id2), toReversePaddedBigInt(id4))
})
it('can sort pleroma ids', () => {
// these are already in base62 sorted order
let ids = [
'9gP7cpqqJWyp93GxRw',
@ -126,4 +203,88 @@ describe('test-id-sorting.js', () => {
gt(toReversePaddedBigInt(prev), toReversePaddedBigInt(next))
}
})
it('can sort pleroma ids 2', () => {
let ids = [
'9gTv5mTEiXL6ZpqYHg',
'9gTv5mK1Gny07FXBuy',
'9gTv5kXlshmKbJx94i',
'9gTv5f9TvaNVsXYB1M',
'9gTv5Fj2SlN71HzCpk',
'9gTv5DibvNqCnNlptA',
'9gTv4ttvbb0hKguaki',
'9gTv4n17CTbFyxDjU0',
'9gTv43wGndzCAtbjBQ',
'9gTv3zP9ep7W725E0m',
'9gTv3mQRuhQnrHaZiy',
'9gTv3mOK2bjJkgasPQ',
'9gTv3kpXqQJiuXJaYy',
'9gTv3JliSYDAyqJeLY',
'9gTv36jSvbgeY6AAXA',
'9gTv2udLuVfP1fD7L6',
'9gTv2gbQ4tnnVcCNNo',
'9gTv2FSH0nRXJQiBsW',
'9gTv1tfzz5LllcxqDY',
'9gTv1t1EQejxjBtHfs'
].reverse()
for (let i = 1; i < ids.length; i++) {
let prev = ids[i - 1]
let next = ids[i]
lt(toPaddedBigInt(prev), toPaddedBigInt(next))
gt(toReversePaddedBigInt(prev), toReversePaddedBigInt(next))
}
})
it('can sort pleroma ids - 3', () => {
let ids = [
'9gTCUO2xe7vfkLbHRA',
'9gT35b559J1tLPhGj2',
'9gRax4YxAwDuIdr83U',
'9gQqktJiZ6ha3Tz0fA',
'9gOyRmT0DkfWcmnw7k',
'9gOvJd0nBsvd7a1wUy',
'9gOsyLgqCKARPQQKPo',
'9gOsiEkAFyEnERINOq',
'9gOsHBDvEh0EUoJRom',
'9gOrx9MURrsivilkjw',
'9gOrpQWsjga3rEwmw4',
'9gOriOTFzUcLRjtjkm',
'9gOraOfrLpz3lBynXE',
'9gOrLvdMWe0Ldgudbk',
'9gOrB5vttTXiOaPxCq',
'9gOpJz8uJEGa2Ac47E',
'9gOhCP31JNzWQXjVEu',
'9gOdCdne43SIMZOp8K',
'9gOcJKlz6VfvQRFZyq',
'9gOcGtLW6bmXmchoGG',
'9gOc78e2N27GKriYdM',
'9gOc4BmYFXBvCHIGzw',
'9gObs3Wx6Rt888DCHQ',
'9gObp0SU4J0tjXfSZk',
'9gOaaY4UiJOngVzN6e',
'9gOaAiTbPDbQQTPnpQ',
'9gOZjQH0yaB29SVrIe',
'9gOZTFEsEbV1IMnUDA',
'9gOZQA9yf55sfs2NYu',
'9gOZNK8p5vaulK8yKO',
'9gOZMvOZ6GKe0uJgtU',
'9gOZIXp4ndv9kMd1SC',
'9gOWmmfFsr2obOlJuy',
'9gOWivHDvAoO9egFmK',
'9gOWZDwXfIAH9TNkbQ',
'9gOWUUUkhMq92R7v6m',
'9gOWChzRml0fgE0MJk',
'9gOVO8G0MitkNRdIjw',
'9gNEd8FWme8oJn15Jg',
'9gNEUoR2ZKkgY4Qzce'
].reverse()
for (let i = 1; i < ids.length; i++) {
let prev = ids[i - 1]
let next = ids[i]
lt(toPaddedBigInt(prev), toPaddedBigInt(next))
gt(toReversePaddedBigInt(prev), toReversePaddedBigInt(next))
}
})
})