From cc5cb3150dcb48434df815d0f35e2ce6619487b4 Mon Sep 17 00:00:00 2001 From: Alina Sireneva Date: Thu, 8 Feb 2024 02:07:47 +0300 Subject: [PATCH] fix(sqlite): added migrations for older storage schema --- .../dispatcher/src/state/providers/sqlite.ts | 4 ++ packages/sqlite/package.json | 1 + packages/sqlite/src/driver.ts | 35 ++++++++-- packages/sqlite/src/repository/auth-keys.ts | 8 ++- packages/sqlite/src/repository/kv.ts | 64 +++++++++++++++++++ packages/sqlite/src/repository/peers.ts | 4 ++ .../sqlite/src/repository/ref-messages.ts | 10 +-- pnpm-lock.yaml | 3 + 8 files changed, 116 insertions(+), 13 deletions(-) diff --git a/packages/dispatcher/src/state/providers/sqlite.ts b/packages/dispatcher/src/state/providers/sqlite.ts index e7361893..38ebc5de 100644 --- a/packages/dispatcher/src/state/providers/sqlite.ts +++ b/packages/dispatcher/src/state/providers/sqlite.ts @@ -43,6 +43,10 @@ class SqliteStateRepository implements IStateRepository { this._deleteRl = _driver.db.prepare('delete from rl_state where key = ?') this._deleteOldRl = _driver.db.prepare('delete from rl_state where reset < ?') }) + _driver.registerLegacyMigration('state', (db) => { + // not too important information, just drop the table + db.exec('drop table state') + }) } private _setState!: Statement diff --git a/packages/sqlite/package.json b/packages/sqlite/package.json index 41a46c37..01a6d131 100644 --- a/packages/sqlite/package.json +++ b/packages/sqlite/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@mtcute/core": "workspace:^", + "@mtcute/tl": "*", "@mtcute/tl-runtime": "workspace:^", "better-sqlite3": "9.2.2" }, diff --git a/packages/sqlite/src/driver.ts b/packages/sqlite/src/driver.ts index fadedbbb..a6c2d60a 100644 --- a/packages/sqlite/src/driver.ts +++ b/packages/sqlite/src/driver.ts @@ -1,6 +1,6 @@ import sqlite3, { Database, Options, Statement } from 'better-sqlite3' -import { BaseStorageDriver, MtUnsupportedError } from '@mtcute/core' +import { BaseStorageDriver } from '@mtcute/core' import { beforeExit } from '@mtcute/core/utils.js' export interface SqliteStorageDriverOptions { @@ -49,6 +49,17 @@ export class SqliteStorageDriver extends BaseStorageDriver { private _migrations: Map> = new Map() private _maxVersion: Map = new Map() + // todo: remove in 1.0.0 + remove direct dep on @mtcute/tl + private _legacyMigrations: Map = new Map() + + registerLegacyMigration(repo: string, migration: MigrationFunction): void { + if (this.loaded) { + throw new Error('Cannot register migrations after loading') + } + + this._legacyMigrations.set(repo, migration) + } + registerMigration(repo: string, version: number, migration: MigrationFunction): void { if (this.loaded) { throw new Error('Cannot register migrations after loading') @@ -88,16 +99,16 @@ export class SqliteStorageDriver extends BaseStorageDriver { this._pending.push([stmt, params]) } + private _runLegacyMigrations = false + _initialize(): void { const hasLegacyTables = this.db .prepare("select name from sqlite_master where type = 'table' and name = 'kv'") .get() if (hasLegacyTables) { - throw new MtUnsupportedError( - 'This database was created with an older version of mtcute, and cannot be used anymore. ' + - 'Please delete the database and try again.', - ) + this._log.info('legacy tables detected, will run migrations') + this._runLegacyMigrations = true } this.db.exec(MIGRATIONS_TABLE_SQL) @@ -154,12 +165,24 @@ export class SqliteStorageDriver extends BaseStorageDriver { }) }) - this._initialize() + this.db.transaction(() => this._initialize())() + this._cleanup = beforeExit(() => { this._save() this._destroy() }) for (const cb of this._onLoad) cb(this.db) + + if (this._runLegacyMigrations) { + this.db.transaction(() => { + for (const migration of this._legacyMigrations.values()) { + migration(this.db) + } + + // in case _writeLater was used + this._runMany(this._pending) + })() + } } _save(): void { diff --git a/packages/sqlite/src/repository/auth-keys.ts b/packages/sqlite/src/repository/auth-keys.ts index affe5699..06b4e2fd 100644 --- a/packages/sqlite/src/repository/auth-keys.ts +++ b/packages/sqlite/src/repository/auth-keys.ts @@ -18,11 +18,11 @@ export class SqliteAuthKeysRepository implements IAuthKeysRepository { constructor(readonly _driver: SqliteStorageDriver) { _driver.registerMigration('auth_keys', 1, (db) => { db.exec(` - create table auth_keys ( + create table if not exists auth_keys ( dc integer primary key, key blob not null ); - create table temp_auth_keys ( + create table if not exists temp_auth_keys ( dc integer not null, idx integer not null, key blob not null, @@ -36,7 +36,9 @@ export class SqliteAuthKeysRepository implements IAuthKeysRepository { this._getTemp = db.prepare('select key from temp_auth_keys where dc = ? and idx = ? and expires > ?') this._set = db.prepare('insert or replace into auth_keys (dc, key) values (?, ?)') - this._setTemp = this._driver.db.prepare('insert or replace into temp_auth_keys (dc, idx, key, expires) values (?, ?, ?, ?)') + this._setTemp = this._driver.db.prepare( + 'insert or replace into temp_auth_keys (dc, idx, key, expires) values (?, ?, ?, ?)', + ) this._del = db.prepare('delete from auth_keys where dc = ?') this._delTemp = db.prepare('delete from temp_auth_keys where dc = ? and idx = ?') diff --git a/packages/sqlite/src/repository/kv.ts b/packages/sqlite/src/repository/kv.ts index 01cd347a..b4d56b14 100644 --- a/packages/sqlite/src/repository/kv.ts +++ b/packages/sqlite/src/repository/kv.ts @@ -1,6 +1,12 @@ import { Statement } from 'better-sqlite3' import { IKeyValueRepository } from '@mtcute/core' +import { CurrentUserService } from '@mtcute/core/src/highlevel/storage/service/current-user.js' +import { UpdatesStateService } from '@mtcute/core/src/highlevel/storage/service/updates.js' +import { ServiceOptions } from '@mtcute/core/src/storage/service/base.js' +import { DefaultDcsService } from '@mtcute/core/src/storage/service/default-dcs.js' +import { __tlReaderMap } from '@mtcute/tl/binary/reader.js' +import { __tlWriterMap } from '@mtcute/tl/binary/writer.js' import { SqliteStorageDriver } from '../driver.js' @@ -25,6 +31,64 @@ export class SqliteKeyValueRepository implements IKeyValueRepository { this._del = db.prepare('delete from key_value where key = ?') this._delAll = db.prepare('delete from key_value') }) + + // awkward dependencies, unsafe code, awful crutches + // all in the name of backwards compatibility + /* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any */ + /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-floating-promises */ + /* eslint-disable @typescript-eslint/no-unsafe-argument */ + _driver.registerLegacyMigration('kv', (db) => { + // fetch all values from the old table + const all = db.prepare('select key, value from kv').all() as { key: string; value: string }[] + const obj: Record = {} + + for (const { key, value } of all) { + obj[key] = JSON.parse(value) + } + + db.exec('drop table kv') + + // lol + const options: ServiceOptions = { + driver: this._driver, + readerMap: __tlReaderMap, + writerMap: __tlWriterMap, + // eslint-disable-next-line dot-notation + log: this._driver['_log'], + } + + if (obj.self) { + new CurrentUserService(this, options).store({ + userId: obj.self.userId, + isBot: obj.self.isBot, + isPremium: false, + usernames: [], + }) + } + + if (obj.pts) { + const svc = new UpdatesStateService(this, options) + svc.setPts(obj.pts) + if (obj.qts) svc.setQts(obj.qts) + if (obj.date) svc.setDate(obj.date) + if (obj.seq) svc.setSeq(obj.seq) + + // also fetch channel states. they were moved to kv from a separate table + const channels = db.prepare('select * from pts').all() as any[] + + for (const channel of channels) { + svc.setChannelPts(channel.channel_id, channel.pts) + } + } + db.exec('drop table pts') + + if (obj.def_dc) { + new DefaultDcsService(this, options).store(obj.def_dc) + } + }) + /* eslint-enable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any */ + /* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-floating-promises */ + /* eslint-enable @typescript-eslint/no-unsafe-argument */ } private _set!: Statement diff --git a/packages/sqlite/src/repository/peers.ts b/packages/sqlite/src/repository/peers.ts index cab6e23e..54a527d4 100644 --- a/packages/sqlite/src/repository/peers.ts +++ b/packages/sqlite/src/repository/peers.ts @@ -54,6 +54,10 @@ export class SqlitePeersRepository implements IPeersRepository { this._delAll = db.prepare('delete from peers') }) + _driver.registerLegacyMigration('peers', (db) => { + // not too important information, just drop the table + db.exec('drop table entities') + }) } private _store!: Statement diff --git a/packages/sqlite/src/repository/ref-messages.ts b/packages/sqlite/src/repository/ref-messages.ts index 9c139148..af4332af 100644 --- a/packages/sqlite/src/repository/ref-messages.ts +++ b/packages/sqlite/src/repository/ref-messages.ts @@ -14,17 +14,19 @@ export class SqliteRefMessagesRepository implements IReferenceMessagesRepository constructor(readonly _driver: SqliteStorageDriver) { _driver.registerMigration('ref_messages', 1, (db) => { db.exec(` - create table message_refs ( + create table if not exists message_refs ( peer_id integer not null, chat_id integer not null, msg_id integer not null ); - create index idx_message_refs_peer on message_refs (peer_id); - create index idx_message_refs on message_refs (chat_id, msg_id); + create index if not exists idx_message_refs_peer on message_refs (peer_id); + create index if not exists idx_message_refs on message_refs (chat_id, msg_id); `) }) _driver.onLoad(() => { - this._store = this._driver.db.prepare('insert or replace into message_refs (peer_id, chat_id, msg_id) values (?, ?, ?)') + this._store = this._driver.db.prepare( + 'insert or replace into message_refs (peer_id, chat_id, msg_id) values (?, ?, ?)', + ) this._getByPeer = this._driver.db.prepare('select chat_id, msg_id from message_refs where peer_id = ?') diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17cb9089..44cf6bcd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -279,6 +279,9 @@ importers: '@mtcute/core': specifier: workspace:^ version: link:../core + '@mtcute/tl': + specifier: '*' + version: link:../tl '@mtcute/tl-runtime': specifier: workspace:^ version: link:../tl-runtime