teidesu/mtcute
1
0
Fork 0
Code Issues 15 Pull requests 1 Projects Releases Packages Wiki Activity Actions

refactor!: large refactor of storage implementation

Browse source 
breaking: pretty much the entire storage thing has been overhauled.
migrations from older versions **are not** available, please do them manually through string sessions
This commit is contained in:
alina 🌸 2024-01-04 00:22:26 +03:00
parent 710c040f60
commit eca99a7535
Signed by: teidesu
SSH key fingerprint: SHA256:uNeCpw6aTSU4aIObXLvHfLkDa82HWH9EiOj9AXOIRpI
107 changed files with 3169 additions and 4129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -61,7 +61,7 @@
"ts-node": "10.9.1",
"tsconfig-paths": "4.2.0",
"typedoc": "0.25.3",
"typescript": "5.0.4",
"typescript": "5.1.6",
"vite": "5.0.3",
"vitest": "0.34.6"
},

23
packages/client/src/client.ts
View file

@ -4,7 +4,7 @@
import {
BaseTelegramClient,
BaseTelegramClientOptions,
ITelegramStorage,
IMtStorageProvider,
Long,
MaybeArray,
MaybeAsync,
@ -12,10 +12,10 @@ import {
PartialOnly,
tl,
} from '@mtcute/core'
import { MemoryStorage } from '@mtcute/core/src/storage/memory.js'
import { MemoryStorage } from '@mtcute/core/src/storage/providers/memory/index.js'
import { tdFileId } from '@mtcute/file-id'
import { AuthState, getAuthState, isSelfPeer, setupAuthState } from './methods/auth/_state.js'
import { isSelfPeer } from './methods/auth/_state.js'
import { checkPassword } from './methods/auth/check-password.js'
import { getPasswordHint } from './methods/auth/get-password-hint.js'
import { logOut } from './methods/auth/log-out.js'
@ -344,7 +344,7 @@ interface TelegramClientOptions extends Omit<BaseTelegramClientOptions, 'storage
*
* If omitted, {@link MemoryStorage} is used
*/
storage?: string | ITelegramStorage
storage?: string | IMtStorageProvider
/**
* Parameters for updates manager.
@ -551,15 +551,6 @@ export interface TelegramClient extends BaseTelegramClient {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
on(name: string, handler: (...args: any[]) => void): this
/**
* Get auth state for the given client, containing
* information about the current user.
*
* Auth state must first be initialized with {@link setupAuthState}.
* **Available**: ✅ both users and bots
*
*/
getAuthState(): AuthState
/**
* Check if the given peer/input peer is referring to the current user
* **Available**: ✅ both users and bots
@ -5299,16 +5290,10 @@ export class TelegramClient extends BaseTelegramClient {
},
}),
})
} else {
setupAuthState(this)
}
}
}
TelegramClient.prototype.getAuthState = function (...args) {
return getAuthState(this, ...args)
}
TelegramClient.prototype.isSelfPeer = function (...args) {
return isSelfPeer(this, ...args)
}

2
packages/client/src/methods/_imports.ts
View file

@ -3,7 +3,7 @@
import {
BaseTelegramClient,
BaseTelegramClientOptions,
ITelegramStorage,
IMtStorageProvider,
Long,
MaybeArray,
MaybeAsync,

10
packages/client/src/methods/_init.ts
View file

@ -1,8 +1,8 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
import { BaseTelegramClientOptions, ITelegramStorage } from '@mtcute/core'
import { BaseTelegramClientOptions, IMtStorageProvider } from '@mtcute/core'
// @copy
import { MemoryStorage } from '@mtcute/core/src/storage/memory.js'
import { MemoryStorage } from '@mtcute/core/src/storage/providers/memory/index.js'
import { TelegramClient } from '../client.js'
// @copy
@ -10,8 +10,6 @@ import { Conversation } from '../types/conversation.js'
// @copy
import { _defaultStorageFactory } from '../utils/platform/storage.js'
// @copy
import { setupAuthState } from './auth/_state.js'
// @copy
import {
enableUpdatesProcessing,
makeParsedUpdateHandler,
@ -35,7 +33,7 @@ interface TelegramClientOptions extends Omit<BaseTelegramClientOptions, 'storage
*
* If omitted, {@link MemoryStorage} is used
*/
storage?: string | ITelegramStorage
storage?: string | IMtStorageProvider
/**
* Parameters for updates manager.
@ -109,7 +107,5 @@ function _initializeClient(this: TelegramClient, opts: TelegramClientOptions) {
},
}),
})
} else {
setupAuthState(this)
}
}

102
packages/client/src/methods/auth/_state.ts
View file

@ -1,100 +1,13 @@
/* eslint-disable no-inner-declarations */
import { BaseTelegramClient, MtArgumentError, MtUnsupportedError, tl } from '@mtcute/core'
import { BaseTelegramClient, MtUnsupportedError, tl } from '@mtcute/core'
import { assertTypeIs } from '@mtcute/core/utils.js'
import { User } from '../../types/peers/user.js'
const STATE_SYMBOL = Symbol('authState')
/** @exported */
export interface AuthState {
// local copy of "self" in storage,
// so we can use it w/out relying on storage.
// they are both loaded and saved to storage along with the updates
// (see methods/updates)
userId: number | null
isBot: boolean
selfUsername: string | null
selfChanged?: boolean
}
/**
* Initialize auth state for the given client.
*
* Allows {@link getAuthState} to be used and is required for some methods.
* @noemit
*/
export function setupAuthState(client: BaseTelegramClient): void {
// eslint-disable-next-line
let state: AuthState = (client as any)[STATE_SYMBOL]
if (state) return
// init
// eslint-disable-next-line @typescript-eslint/no-explicit-any
state = (client as any)[STATE_SYMBOL] = {
userId: null,
isBot: false,
selfUsername: null,
}
client.log.prefix = '[USER N/A] '
function onBeforeConnect() {
Promise.resolve(client.storage.getSelf())
.then((self) => {
if (!self) return
state.userId = self.userId
state.isBot = self.isBot
client.log.prefix = `[USER ${self.userId}] `
})
.catch((err) => client._emitError(err))
}
async function onBeforeStorageSave() {
if (state.selfChanged) {
await client.storage.setSelf(
state.userId ?
{
userId: state.userId,
isBot: state.isBot,
} :
null,
)
state.selfChanged = false
}
}
client.on('before_connect', onBeforeConnect)
client.beforeStorageSave(onBeforeStorageSave)
client.on('before_stop', () => {
client.off('before_connect', onBeforeConnect)
client.offBeforeStorageSave(onBeforeStorageSave)
})
}
/**
* Get auth state for the given client, containing
* information about the current user.
*
* Auth state must first be initialized with {@link setupAuthState}.
*/
export function getAuthState(client: BaseTelegramClient): AuthState {
// eslint-disable-next-line
let state: AuthState = (client as any)[STATE_SYMBOL]
if (!state) {
throw new MtArgumentError('Auth state is not initialized, use setupAuthState()')
}
return state
}
/** @internal */
export async function _onAuthorization(
client: BaseTelegramClient,
auth: tl.auth.TypeAuthorization,
bot = false,
): Promise<User> {
if (auth._ === 'auth.authorizationSignUpRequired') {
throw new MtUnsupportedError(
@ -104,14 +17,8 @@ export async function _onAuthorization(
assertTypeIs('_onAuthorization (@ auth.authorization -> user)', auth.user, 'user')
const state = getAuthState(client)
state.userId = auth.user.id
state.isBot = bot
state.selfUsername = auth.user.username ?? null
state.selfChanged = true
client.notifyLoggedIn(auth)
await client.saveStorage()
// todo: selfUsername
await client.notifyLoggedIn(auth)
// telegram ignores invokeWithoutUpdates for auth methods
if (client.network.params.disableUpdates) client.network.resetSessions()
@ -126,7 +33,8 @@ export function isSelfPeer(
client: BaseTelegramClient,
peer: tl.TypeInputPeer | tl.TypePeer | tl.TypeInputUser,
): boolean {
const state = getAuthState(client)
const state = client.storage.self.getCached()
if (!state) return false
switch (peer._) {
case 'inputPeerSelf':

12
packages/client/src/methods/auth/log-out.ts
View file

@ -1,7 +1,5 @@
import { BaseTelegramClient } from '@mtcute/core'
import { getAuthState } from './_state.js'
/**
* Log out from Telegram account and optionally reset the session storage.
*
@ -13,16 +11,12 @@ import { getAuthState } from './_state.js'
export async function logOut(client: BaseTelegramClient): Promise<true> {
await client.call({ _: 'auth.logOut' })
const authState = getAuthState(client)
authState.userId = null
authState.isBot = false
authState.selfUsername = null
authState.selfChanged = true
await client.storage.self.store(null)
// authState.selfUsername = null todo
client.emit('logged_out')
await client.storage.reset()
await client.saveStorage()
await client.storage.clear()
return true
}

2
packages/client/src/methods/auth/sign-in-bot.ts
View file

@ -19,5 +19,5 @@ export async function signInBot(client: BaseTelegramClient, token: string): Prom
botAuthToken: token,
})
return _onAuthorization(client, res, true)
return _onAuthorization(client, res)
}

3
packages/client/src/methods/chats/batched-queries.ts
View file

@ -8,7 +8,6 @@ import {
toInputUser,
} from '../../utils/peer-utils.js'
import { batchedQuery } from '../../utils/query-batcher.js'
import { getAuthState } from '../auth/_state.js'
/** @internal */
export const _getUsersBatched = batchedQuery<tl.TypeInputUser, tl.TypeUser, number>({
@ -27,7 +26,7 @@ export const _getUsersBatched = batchedQuery<tl.TypeInputUser, tl.TypeUser, numb
case 'inputUserFromMessage':
return item.userId
case 'inputUserSelf':
return getAuthState(client).userId!
return client.storage.self.getCached()!.userId
default:
throw new MtArgumentError('Invalid input user')
}

3
packages/client/src/methods/messages/get-messages.ts
View file

@ -4,7 +4,6 @@ import { assertTypeIsNot } from '@mtcute/core/utils.js'
import { Message } from '../../types/messages/index.js'
import { InputPeerLike, PeersIndex } from '../../types/peers/index.js'
import { isInputPeerChannel, toInputChannel } from '../../utils/peer-utils.js'
import { getAuthState } from '../auth/_state.js'
import { resolvePeer } from '../users/resolve-peer.js'
// @available=both
@ -64,7 +63,7 @@ export async function getMessages(
// (channels have their own message numbering)
switch (peer._) {
case 'inputPeerSelf':
if (selfId === undefined) selfId = getAuthState(client).userId
if (selfId === undefined) selfId = client.storage.self.getCached()?.userId ?? null
if (!(msg.peerId._ === 'peerUser' && msg.peerId.userId === selfId)) {
return null

7
packages/client/src/methods/messages/send-text.test.ts
View file

@ -3,7 +3,6 @@ import { describe, expect, it, vi } from 'vitest'
import { Long, toggleChannelIdMark } from '@mtcute/core'
import { createStub, StubTelegramClient } from '@mtcute/test'
import { getAuthState, setupAuthState } from '../auth/_state.js'
import { sendText } from './send-text.js'
const stubUser = createStub('user', {
@ -103,9 +102,8 @@ describe('sendText', () => {
it('should correctly handle updateShortSentMessage with cached peer', async () => {
const client = new StubTelegramClient()
client.storage.self.store({ userId: stubUser.id, isBot: false })
await client.registerPeers(stubUser)
setupAuthState(client)
getAuthState(client).userId = stubUser.id
client.respondWith('messages.sendMessage', () =>
createStub('updateShortSentMessage', {
@ -128,8 +126,7 @@ describe('sendText', () => {
it('should correctly handle updateShortSentMessage without cached peer', async () => {
const client = new StubTelegramClient()
setupAuthState(client)
getAuthState(client).userId = stubUser.id
client.storage.self.store({ userId: stubUser.id, isBot: false })
const getUsersFn = client.respondWith(
'users.getUsers',

5
packages/client/src/methods/messages/send-text.ts
View file

@ -7,7 +7,6 @@ import { InputText } from '../../types/misc/entities.js'
import { InputPeerLike, PeersIndex } from '../../types/peers/index.js'
import { inputPeerToPeer } from '../../utils/peer-utils.js'
import { createDummyUpdate } from '../../utils/updates-utils.js'
import { getAuthState } from '../auth/_state.js'
import { _getRawPeerBatched } from '../chats/batched-queries.js'
import { _normalizeInputText } from '../misc/normalize-text.js'
import { resolvePeer } from '../users/resolve-peer.js'
@ -80,7 +79,7 @@ export async function sendText(
_: 'message',
id: res.id,
peerId: inputPeerToPeer(peer),
fromId: { _: 'peerUser', userId: getAuthState(client).userId! },
fromId: { _: 'peerUser', userId: client.storage.self.getCached()!.userId },
message,
date: res.date,
out: res.out,
@ -97,7 +96,7 @@ export async function sendText(
const fetchPeer = async (peer: tl.TypePeer | tl.TypeInputPeer): Promise<void> => {
const id = getMarkedPeerId(peer)
let cached = await client.storage.getFullPeerById(id)
let cached = await client.storage.peers.getCompleteById(id)
if (!cached) {
cached = await _getRawPeerBatched(client, await resolvePeer(client, peer))

4
packages/client/src/methods/misc/chain-id.ts
View file

@ -1,10 +1,8 @@
import { BaseTelegramClient, getMarkedPeerId, tl } from '@mtcute/core'
import { getAuthState } from '../auth/_state.js'
/** @internal */
export function _getPeerChainId(client: BaseTelegramClient, peer: tl.TypeInputPeer, prefix = 'peer') {
const id = peer._ === 'inputPeerSelf' ? getAuthState(client).userId! : getMarkedPeerId(peer)
const id = peer._ === 'inputPeerSelf' ? client.storage.self.getCached()!.userId : getMarkedPeerId(peer)
return `${prefix}:${id}`
}

61
packages/client/src/methods/updates/manager.ts
View file

@ -1,12 +1,11 @@
/* eslint-disable max-depth,max-params */
import { assertNever, BaseTelegramClient, MaybeAsync, MtArgumentError, tl } from '@mtcute/core'
import { getBarePeerId, getMarkedPeerId, markedPeerIdToBare, toggleChannelIdMark } from '@mtcute/core/utils.js'
import { assertNever, BaseTelegramClient, MaybeAsync, MtArgumentError, parseMarkedPeerId, tl } from '@mtcute/core'
import { getBarePeerId, getMarkedPeerId, toggleChannelIdMark } from '@mtcute/core/utils.js'
import { PeersIndex } from '../../types/index.js'
import { isInputPeerChannel, isInputPeerUser, toInputChannel, toInputUser } from '../../utils/peer-utils.js'
import { RpsMeter } from '../../utils/rps-meter.js'
import { createDummyUpdatesContainer } from '../../utils/updates-utils.js'
import { getAuthState, setupAuthState } from '../auth/_state.js'
import { _getChannelsBatched, _getUsersBatched } from '../chats/batched-queries.js'
import { resolvePeer } from '../users/resolve-peer.js'
import { createUpdatesState, PendingUpdate, toPendingUpdate, UpdatesManagerParams, UpdatesState } from './types.js'
@ -84,19 +83,18 @@ export function getCurrentRpsProcessing(client: BaseTelegramClient): number {
export function enableUpdatesProcessing(client: BaseTelegramClient, params: UpdatesManagerParams): void {
if (getState(client)) return
setupAuthState(client)
if (client.network.params.disableUpdates) {
throw new MtArgumentError('Updates must be enabled to use updates manager')
}
const authState = getAuthState(client)
const authState = client.storage.self.getCached(true)
const state = createUpdatesState(client, authState, params)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(client as any)[STATE_SYMBOL] = state
function onLoggedIn(): void {
state.auth = client.storage.self.getCached()
fetchUpdatesState(client, state).catch((err) => client._emitError(err))
}
@ -111,10 +109,6 @@ export function enableUpdatesProcessing(client: BaseTelegramClient, params: Upda
loadUpdatesStorage(client, state).catch((err) => client._emitError(err))
}
function onBeforeStorageSave(): Promise<void> {
return saveUpdatesStorage(client, state).catch((err) => client._emitError(err))
}
function onKeepAlive() {
state.log.debug('no updates for >15 minutes, catching up')
handleUpdate(state, { _: 'updatesTooLong' })
@ -128,7 +122,6 @@ export function enableUpdatesProcessing(client: BaseTelegramClient, params: Upda
client.on('logged_in', onLoggedIn)
client.on('logged_out', onLoggedOut)
client.on('before_connect', onBeforeConnect)
client.beforeStorageSave(onBeforeStorageSave)
client.on('keep_alive', onKeepAlive)
client.network.setUpdateHandler((upd, fromClient) => handleUpdate(state, upd, fromClient))
@ -136,7 +129,6 @@ export function enableUpdatesProcessing(client: BaseTelegramClient, params: Upda
client.off('logged_in', onLoggedIn)
client.off('logged_out', onLoggedOut)
client.off('before_connect', onBeforeConnect)
client.offBeforeStorageSave(onBeforeStorageSave)
client.off('keep_alive', onKeepAlive)
client.off('before_stop', cleanup)
client.network.setUpdateHandler(() => {})
@ -380,7 +372,7 @@ async function fetchUpdatesState(client: BaseTelegramClient, state: UpdatesState
}
async function loadUpdatesStorage(client: BaseTelegramClient, state: UpdatesState): Promise<void> {
const storedState = await client.storage.getUpdatesState()
const storedState = await client.storage.updates.getState()
if (storedState) {
state.pts = state.oldPts = storedState[0]
@ -406,16 +398,16 @@ async function saveUpdatesStorage(client: BaseTelegramClient, state: UpdatesStat
if (state.pts !== undefined) {
// if old* value is not available, assume it has changed.
if (state.oldPts === undefined || state.oldPts !== state.pts) {
await client.storage.setUpdatesPts(state.pts)
await client.storage.updates.setPts(state.pts)
}
if (state.oldQts === undefined || state.oldQts !== state.qts) {
await client.storage.setUpdatesQts(state.qts!)
await client.storage.updates.setQts(state.qts!)
}
if (state.oldDate === undefined || state.oldDate !== state.date) {
await client.storage.setUpdatesDate(state.date!)
await client.storage.updates.setDate(state.date!)
}
if (state.oldSeq === undefined || state.oldSeq !== state.seq) {
await client.storage.setUpdatesSeq(state.seq!)
await client.storage.updates.setSeq(state.seq!)
}
// update old* values
@ -424,7 +416,7 @@ async function saveUpdatesStorage(client: BaseTelegramClient, state: UpdatesStat
state.oldDate = state.date
state.oldSeq = state.seq
await client.storage.setManyChannelPts(state.cptsMod)
await client.storage.updates.setManyChannelPts(state.cptsMod)
state.cptsMod.clear()
if (save) {
@ -505,7 +497,7 @@ async function fetchMissingPeers(
async function fetchPeer(peer?: tl.TypePeer | number) {
if (!peer) return true
const bare = typeof peer === 'number' ? markedPeerIdToBare(peer) : getBarePeerId(peer)
const bare = typeof peer === 'number' ? parseMarkedPeerId(peer)[1] : getBarePeerId(peer)
const marked = typeof peer === 'number' ? peer : getMarkedPeerId(peer)
const index = marked > 0 ? peers.users : peers.chats
@ -513,7 +505,7 @@ async function fetchMissingPeers(
if (index.has(bare)) return true
if (missing.has(marked)) return false
const cached = await client.storage.getFullPeerById(marked)
const cached = await client.storage.peers.getCompleteById(marked)
if (!cached) {
missing.add(marked)
@ -641,7 +633,7 @@ async function storeMessageReferences(client: BaseTelegramClient, msg: tl.TypeMe
const marked = typeof peer === 'number' ? peer : getMarkedPeerId(peer)
promises.push(client.storage.saveReferenceMessage(marked, channelId, msg.id))
promises.push(client.storage.refMsgs.store(marked, channelId, msg.id))
}
// reference: https://github.com/tdlib/td/blob/master/td/telegram/MessagesManager.cpp
@ -762,7 +754,7 @@ async function fetchChannelDifference(
let _pts: number | null | undefined = state.cpts.get(channelId)
if (!_pts && state.catchUpChannels) {
_pts = await client.storage.getChannelPts(channelId)
_pts = await client.storage.updates.getChannelPts(channelId)
}
if (!_pts) _pts = fallbackPts
@ -782,7 +774,7 @@ async function fetchChannelDifference(
// to make TS happy
let pts = _pts
let limit = state.auth.isBot ? 100000 : 100
let limit = state.auth?.isBot ? 100000 : 100
if (pts <= 0) {
pts = 1
@ -1182,27 +1174,28 @@ async function onUpdate(
client.network.config.update(true).catch((err) => client._emitError(err))
break
case 'updateUserName':
if (upd.userId === state.auth.userId) {
state.auth.selfUsername = upd.usernames.find((it) => it.active)?.username ?? null
}
// todo
// if (upd.userId === state.auth?.userId) {
// state.auth.selfUsername = upd.usernames.find((it) => it.active)?.username ?? null
// }
break
case 'updateDeleteChannelMessages':
if (!state.auth.isBot) {
await client.storage.deleteReferenceMessages(toggleChannelIdMark(upd.channelId), upd.messages)
if (!state.auth?.isBot) {
await client.storage.refMsgs.delete(toggleChannelIdMark(upd.channelId), upd.messages)
}
break
case 'updateNewMessage':
case 'updateEditMessage':
case 'updateNewChannelMessage':
case 'updateEditChannelMessage':
if (!state.auth.isBot) {
if (!state.auth?.isBot) {
await storeMessageReferences(client, upd.message)
}
break
}
if (missing?.size) {
if (state.auth.isBot) {
if (state.auth?.isBot) {
state.log.warn(
'missing peers (%J) after getDifference for %s (pts = %d, cid = %d)',
missing,
@ -1215,7 +1208,7 @@ async function onUpdate(
await client.storage.save?.()
for (const id of missing) {
Promise.resolve(client.storage.getPeerById(id))
Promise.resolve(client.storage.peers.getById(id))
.then((peer): unknown => {
if (!peer) {
state.log.warn('cannot fetch full peer %d - getPeerById returned null', id)
@ -1341,7 +1334,7 @@ async function updatesLoop(client: BaseTelegramClient, state: UpdatesState): Pro
log.debug('received %s (size = %d)', upd._, upd.updates.length)
}
await client._cachePeersFrom(upd)
await client.storage.peers.updatePeersFrom(upd)
const peers = PeersIndex.from(upd)
@ -1422,7 +1415,7 @@ async function updatesLoop(client: BaseTelegramClient, state: UpdatesState): Pro
id: upd.id,
fromId: {
_: 'peerUser',
userId: upd.out ? state.auth.userId! : upd.userId,
userId: upd.out ? state.auth!.userId : upd.userId,
},
peerId: {
_: 'peerUser',
@ -1528,7 +1521,7 @@ async function updatesLoop(client: BaseTelegramClient, state: UpdatesState): Pro
// update gaps (i.e. first update received is considered
// to be the base state)
const saved = await client.storage.getChannelPts(pending.channelId)
const saved = await client.storage.updates.getChannelPts(pending.channelId)
if (saved) {
state.cpts.set(pending.channelId, saved)

6
packages/client/src/methods/updates/types.ts
View file

@ -1,9 +1,9 @@
import { BaseTelegramClient, tl } from '@mtcute/core'
import type { CurrentUserInfo } from '@mtcute/core/src/storage/service/current-user.js'
import { AsyncLock, ConditionVariable, Deque, EarlyTimer, Logger, SortedLinkedList } from '@mtcute/core/utils.js'
import { PeersIndex } from '../../types/index.js'
import { RpsMeter } from '../../utils/index.js'
import { AuthState } from '../auth/_state.js'
import { extractChannelIdFromUpdate } from './utils.js'
/**
@ -134,7 +134,7 @@ export interface UpdatesState {
log: Logger
stop: () => void
handler: RawUpdateHandler
auth: AuthState
auth: CurrentUserInfo | null
}
/**
@ -143,7 +143,7 @@ export interface UpdatesState {
*/
export function createUpdatesState(
client: BaseTelegramClient,
authState: AuthState,
authState: CurrentUserInfo | null,
opts: UpdatesManagerParams,
): UpdatesState {
return {

21
packages/client/src/methods/users/get-me.ts
View file

@ -2,7 +2,6 @@ import { BaseTelegramClient } from '@mtcute/core'
import { assertTypeIs } from '@mtcute/core/utils.js'
import { User } from '../../types/index.js'
import { getAuthState } from '../auth/_state.js'
/**
* Get currently authorized user's full information
@ -20,21 +19,13 @@ export function getMe(client: BaseTelegramClient): Promise<User> {
.then(async ([user]) => {
assertTypeIs('getMe (@ users.getUsers)', user, 'user')
const authState = getAuthState(client)
await client.storage.self.store({
userId: user.id,
isBot: user.bot!,
})
if (authState.userId !== user.id) {
// there is such possibility, e.g. when
// using a string session without `self`,
// or logging out and re-logging in
// we need to update the fields accordingly,
// and force-save the session
authState.userId = user.id
authState.isBot = Boolean(user.bot)
authState.selfChanged = true
await client.saveStorage()
}
authState.selfUsername = user.username ?? null
// todo
// authState.selfUsername = user.username ?? null
return new User(user)
})

5
packages/client/src/methods/users/get-my-username.ts
View file

@ -1,7 +1,5 @@
import { BaseTelegramClient } from '@mtcute/core'
import { getAuthState } from '../auth/_state.js'
/**
* Get currently authorized user's username.
*
@ -9,5 +7,6 @@ import { getAuthState } from '../auth/_state.js'
* does not call any API methods.
*/
export function getMyUsername(client: BaseTelegramClient): string | null {
return getAuthState(client).selfUsername
throw new Error('Not implemented')
// return getAuthState(client).selfUsername
}

8
packages/client/src/methods/users/resolve-peer.test.ts
View file

@ -55,8 +55,8 @@ describe('resolvePeer', () => {
accessHash: Long.fromBits(111, 222),
}),
)
await client.storage.saveReferenceMessage(123, -1000000000456, 789)
await client.storage.saveReferenceMessage(-1000000000123, -1000000000456, 789)
await client.storage.refMsgs.store(123, -1000000000456, 789)
await client.storage.refMsgs.store(-1000000000123, -1000000000456, 789)
const resolved = await resolvePeer(client, {
_: 'mtcute.dummyInputPeerMinUser',
@ -124,7 +124,7 @@ describe('resolvePeer', () => {
accessHash: Long.fromBits(111, 222),
}),
)
await client.storage.saveReferenceMessage(123, -1000000000456, 789)
await client.storage.refMsgs.store(123, -1000000000456, 789)
const resolved = await resolvePeer(client, 123)
@ -182,7 +182,7 @@ describe('resolvePeer', () => {
accessHash: Long.fromBits(111, 222),
}),
)
await client.storage.saveReferenceMessage(-1000000000123, -1000000000456, 789)
await client.storage.refMsgs.store(-1000000000123, -1000000000456, 789)
const resolved = await resolvePeer(client, -1000000000123)

21
packages/client/src/methods/users/resolve-peer.ts
View file

@ -1,9 +1,9 @@
import {
BaseTelegramClient,
getBasicPeerType,
getMarkedPeerId,
Long,
MtTypeAssertionError,
parseMarkedPeerId,
tl,
toggleChannelIdMark,
} from '@mtcute/core'
@ -51,7 +51,7 @@ export async function resolvePeer(
}
if (typeof peerId === 'number' && !force) {
const fromStorage = await client.storage.getPeerById(peerId)
const fromStorage = await client.storage.peers.getById(peerId)
if (fromStorage) return fromStorage
}
@ -64,7 +64,7 @@ export async function resolvePeer(
if (peerId.match(/^\d+$/)) {
// phone number
const fromStorage = await client.storage.getPeerByPhone(peerId)
const fromStorage = await client.storage.peers.getByPhone(peerId)
if (fromStorage) return fromStorage
res = await client.call({
@ -74,7 +74,7 @@ export async function resolvePeer(
} else {
// username
if (!force) {
const fromStorage = await client.storage.getPeerByUsername(peerId.toLowerCase())
const fromStorage = await client.storage.peers.getByUsername(peerId)
if (fromStorage) return fromStorage
}
@ -140,28 +140,25 @@ export async function resolvePeer(
// particularly, when we're a bot or we're referencing a user
// who we have "seen" recently
// if it's not the case, we'll get an `PEER_ID_INVALID` error anyways
const peerType = getBasicPeerType(peerId)
const [peerType, bareId] = parseMarkedPeerId(peerId)
switch (peerType) {
case 'user':
return {
_: 'inputPeerUser',
userId: peerId,
userId: bareId,
accessHash: Long.ZERO,
}
case 'chat':
return {
_: 'inputPeerChat',
chatId: -peerId,
chatId: bareId,
}
case 'channel': {
const id = toggleChannelIdMark(peerId)
case 'channel':
return {
_: 'inputPeerChannel',
channelId: id,
channelId: bareId,
accessHash: Long.ZERO,
}
}
}
}

4
packages/client/src/methods/users/set-my-username.ts
View file

@ -1,7 +1,6 @@
import { BaseTelegramClient } from '@mtcute/core'
import { User } from '../../types/index.js'
import { getAuthState } from '../auth/_state.js'
/**
* Change username of the current user.
@ -19,7 +18,8 @@ export async function setMyUsername(client: BaseTelegramClient, username: string
username,
})
getAuthState(client).selfUsername = username || null
// todo
// getAuthState(client).selfUsername = username || null
return new User(res)
}

5
packages/client/src/utils/platform/storage.ts
View file

@ -1,6 +1,5 @@
import { JsonFileStorage } from '@mtcute/core/src/storage/json-file.js'
/** @internal */
export const _defaultStorageFactory = (name: string) => {
return new JsonFileStorage(name)
// todo: move sqlite to core?
throw new Error('Not implemented')
}

2
packages/client/src/utils/platform/storage.web.ts
View file

@ -1,4 +1,4 @@
import { IdbStorage } from '@mtcute/core/src/storage/idb.js'
import { IdbStorage } from '@mtcute/core'
import { MtUnsupportedError } from '../../index.js'

180
packages/core/src/base-client.ts
View file

@ -1,6 +1,5 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import EventEmitter from 'events'
import Long from 'long'
import { tl } from '@mtcute/tl'
import { __tlReaderMap as defaultReaderMap } from '@mtcute/tl/binary/reader.js'
@ -9,24 +8,23 @@ import { TlReaderMap, TlWriterMap } from '@mtcute/tl-runtime'
import { BaseTelegramClientOptions } from './base-client.types.js'
import { ConfigManager } from './network/config-manager.js'
import { SessionConnection, TransportFactory } from './network/index.js'
import { SessionConnection } from './network/index.js'
import { NetworkManager, RpcCallOptions } from './network/network-manager.js'
import { ITelegramStorage } from './storage/index.js'
import { StorageManager } from './storage/storage.js'
import { MustEqual } from './types/index.js'
import {
ControllablePromise,
createControllablePromise,
DcOptions,
defaultCryptoProviderFactory,
defaultProductionDc,
defaultProductionIpv6Dc,
defaultTestDc,
defaultTestIpv6Dc,
getAllPeersFrom,
ICryptoProvider,
LogManager,
readStringSession,
StringSessionData,
toggleChannelIdMark,
writeStringSession,
} from './utils/index.js'
@ -40,10 +38,8 @@ export class BaseTelegramClient extends EventEmitter {
*/
readonly crypto: ICryptoProvider
/**
* Telegram storage taken from {@link BaseTelegramClientOptions.storage}
*/
readonly storage: ITelegramStorage
/** Storage manager */
readonly storage: StorageManager
/**
* "Test mode" taken from {@link BaseTelegramClientOptions.testMode}
@ -54,7 +50,7 @@ export class BaseTelegramClient extends EventEmitter {
* Primary DCs taken from {@link BaseTelegramClientOptions.defaultDcs},
* loaded from session or changed by other means (like redirecting).
*/
protected _defaultDcs: ITelegramStorage.DcOptions
protected _defaultDcs: DcOptions
private _niceStacks: boolean
/** TL layer used by the client */
@ -64,9 +60,6 @@ export class BaseTelegramClient extends EventEmitter {
/** TL writers map used by the client */
readonly _writerMap: TlWriterMap
/** Unix timestamp when the last update was received */
protected _lastUpdateTime = 0
readonly _config = new ConfigManager(() => this.call({ _: 'help.getConfig' }))
// not really connected, but rather "connect() was called"
@ -88,7 +81,6 @@ export class BaseTelegramClient extends EventEmitter {
}
this.crypto = (params.crypto ?? defaultCryptoProviderFactory)()
this.storage = params.storage
this._testMode = Boolean(params.testMode)
let dc = params.defaultDcs
@ -108,6 +100,14 @@ export class BaseTelegramClient extends EventEmitter {
this._readerMap = params.readerMap ?? defaultReaderMap
this._writerMap = params.writerMap ?? defaultWriterMap
this.storage = new StorageManager({
provider: params.storage,
log: this.log,
readerMap: this._readerMap,
writerMap: this._writerMap,
...params.storageOptions,
})
this.network = new NetworkManager(
{
apiId: params.apiId,
@ -127,40 +127,12 @@ export class BaseTelegramClient extends EventEmitter {
maxRetryCount: params.maxRetryCount ?? 5,
isPremium: false,
useIpv6: Boolean(params.useIpv6),
keepAliveAction: this._keepAliveAction.bind(this),
enableErrorReporting: params.enableErrorReporting ?? false,
onUsable: () => this.emit('usable'),
...(params.network ?? {}),
...params.network,
},
this._config,
)
this.storage.setup?.(this.log, this._readerMap, this._writerMap)
}
protected _keepAliveAction(): void {
this.emit('keep_alive')
}
protected async _loadStorage(): Promise<void> {
await this.storage.load?.()
}
_beforeStorageSave: (() => Promise<void>)[] = []
beforeStorageSave(cb: () => Promise<void>): void {
this._beforeStorageSave.push(cb)
}
offBeforeStorageSave(cb: () => Promise<void>): void {
this._beforeStorageSave = this._beforeStorageSave.filter((x) => x !== cb)
}
async saveStorage(): Promise<void> {
for (const cb of this._beforeStorageSave) {
await cb()
}
await this.storage.save?.()
}
/**
@ -180,11 +152,15 @@ export class BaseTelegramClient extends EventEmitter {
const promise = (this._connected = createControllablePromise())
await this.crypto.initialize?.()
await this._loadStorage()
const primaryDc = await this.storage.getDefaultDcs()
await this.storage.load()
const primaryDc = await this.storage.dcs.fetch()
if (primaryDc !== null) this._defaultDcs = primaryDc
const defaultDcAuthKey = await this.storage.getAuthKeyFor(this._defaultDcs.main.id)
const self = await this.storage.self.fetch()
this.log.prefix = `[USER ${self?.userId ?? 'n/a'}] `
const defaultDcAuthKey = await this.storage.provider.authKeys.get(this._defaultDcs.main.id)
if ((this._importForce || !defaultDcAuthKey) && this._importFrom) {
const data = this._importFrom
@ -199,16 +175,15 @@ export class BaseTelegramClient extends EventEmitter {
}
this._defaultDcs = data.primaryDcs
await this.storage.setDefaultDcs(data.primaryDcs)
await this.storage.dcs.store(data.primaryDcs)
if (data.self) {
await this.storage.setSelf(data.self)
await this.storage.self.store(data.self)
}
// await this.primaryConnection.setupKeys(data.authKey)
await this.storage.setAuthKeyFor(data.primaryDcs.main.id, data.authKey)
await this.storage.provider.authKeys.set(data.primaryDcs.main.id, data.authKey)
await this.saveStorage()
await this.storage.save()
}
this.emit('before_connect')
@ -231,7 +206,7 @@ export class BaseTelegramClient extends EventEmitter {
this._config.destroy()
this.network.destroy()
await this.saveStorage()
await this.storage.save()
await this.storage.destroy?.()
this.emit('closed')
@ -262,9 +237,7 @@ export class BaseTelegramClient extends EventEmitter {
const res = await this.network.call(message, params, stack)
if (await this._cachePeersFrom(res)) {
await this.saveStorage()
}
await this.storage.peers.updatePeersFrom(res)
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return res
@ -308,20 +281,6 @@ export class BaseTelegramClient extends EventEmitter {
return this.withCallParams({ abortSignal: signal })
}
/**
* Change transport for the client.
*
* Can be used, for example, to change proxy at runtime
*
* This effectively calls `changeTransport()` on
* `primaryConnection` and all additional connections.
*
* @param factory New transport factory
*/
changeTransport(factory: TransportFactory): void {
this.network.changeTransport(factory)
}
/**
* Register an error handler for the client
*
@ -335,81 +294,16 @@ export class BaseTelegramClient extends EventEmitter {
this._emitError = handler
}
notifyLoggedIn(auth: tl.auth.RawAuthorization): void {
async notifyLoggedIn(auth: tl.auth.RawAuthorization): Promise<void> {
this.network.notifyLoggedIn(auth)
this.log.prefix = `[USER ${auth.user.id}] `
await this.storage.self.store({
userId: auth.user.id,
isBot: auth.user._ === 'user' && auth.user.bot!,
})
this.emit('logged_in', auth)
}
/**
* **ADVANCED**
*
* Adds all peers from a given object to entity cache in storage.
*/
async _cachePeersFrom(obj: object): Promise<boolean> {
const parsedPeers: ITelegramStorage.PeerInfo[] = []
let count = 0
for (const peer of getAllPeersFrom(obj as tl.TlObject)) {
if ((peer as any).min) {
// no point in caching min peers as we can't use them
continue
}
count += 1
switch (peer._) {
case 'user':
if (!peer.accessHash) {
this.log.warn('received user without access hash: %j', peer)
continue
}
parsedPeers.push({
id: peer.id,
accessHash: peer.accessHash,
username: peer.username?.toLowerCase(),
phone: peer.phone,
type: 'user',
full: peer,
})
break
case 'chat':
case 'chatForbidden':
parsedPeers.push({
id: -peer.id,
accessHash: Long.ZERO,
type: 'chat',
full: peer,
})
break
case 'channel':
case 'channelForbidden':
if (!peer.accessHash) {
this.log.warn('received user without access hash: %j', peer)
continue
}
parsedPeers.push({
id: toggleChannelIdMark(peer.id),
accessHash: peer.accessHash,
username: peer._ === 'channel' ? peer.username?.toLowerCase() : undefined,
type: 'channel',
full: peer,
})
break
}
}
if (count > 0) {
await this.storage.updatePeers(parsedPeers)
this.log.debug('cached %d peers', count)
return true
}
return false
}
/**
* Export current session to a single *LONG* string, containing
* all the needed information.
@ -426,14 +320,14 @@ export class BaseTelegramClient extends EventEmitter {
* > with [@BotFather](//t.me/botfather)
*/
async exportSession(): Promise<string> {
const primaryDcs = (await this.storage.getDefaultDcs()) ?? this._defaultDcs
const primaryDcs = (await this.storage.dcs.fetch()) ?? this._defaultDcs
const authKey = await this.storage.getAuthKeyFor(primaryDcs.main.id)
const authKey = await this.storage.provider.authKeys.get(primaryDcs.main.id)
if (!authKey) throw new Error('Auth key is not ready yet')
return writeStringSession(this._writerMap, {
version: 2,
self: await this.storage.getSelf(),
self: await this.storage.self.fetch(),
testMode: this._testMode,
primaryDcs,
authKey,

12
packages/core/src/base-client.types.ts
View file

@ -3,8 +3,9 @@ import { TlReaderMap, TlWriterMap } from '@mtcute/tl-runtime'
import { NetworkManagerExtraParams, ReconnectionStrategy, TransportFactory } from './network/index.js'
import { PersistentConnectionParams } from './network/persistent-connection.js'
import { ITelegramStorage } from './storage/abstract.js'
import { CryptoProviderFactory } from './utils/index.js'
import { IMtStorageProvider } from './storage/provider.js'
import { StorageManagerExtraOptions } from './storage/storage.js'
import { CryptoProviderFactory, DcOptions } from './utils/index.js'
/** Options for {@link BaseTelegramClient} */
export interface BaseTelegramClientOptions {
@ -20,7 +21,10 @@ export interface BaseTelegramClientOptions {
/**
* Storage to use for this client.
*/
storage: ITelegramStorage
storage: IMtStorageProvider
/** Additional options for the storage manager */
storageOptions?: StorageManagerExtraOptions
/**
* Cryptography provider factory to allow delegating
@ -46,7 +50,7 @@ export interface BaseTelegramClientOptions {
*
* @default Production DC 2.
*/
defaultDcs?: ITelegramStorage.DcOptions
defaultDcs?: DcOptions
/**
* Whether to connect to test servers.

50
packages/core/src/network/network-manager.ts
View file

@ -1,9 +1,9 @@
import { mtp, tl } from '@mtcute/tl'
import { TlReaderMap, TlWriterMap } from '@mtcute/tl-runtime'
import { ITelegramStorage } from '../storage/index.js'
import { StorageManager } from '../storage/storage.js'
import { MtArgumentError, MtcuteError, MtTimeoutError } from '../types/index.js'
import { ControllablePromise, createControllablePromise, ICryptoProvider, Logger, sleep } from '../utils/index.js'
import { ControllablePromise, createControllablePromise, DcOptions, ICryptoProvider, Logger, sleep } from '../utils/index.js'
import { assertTypeIs } from '../utils/type-assertions.js'
import { ConfigManager } from './config-manager.js'
import { MultiSessionConnection } from './multi-session-connection.js'
@ -30,7 +30,7 @@ const CLIENT_ERRORS = {
* This type is intended for internal usage only.
*/
export interface NetworkManagerParams {
storage: ITelegramStorage
storage: StorageManager
crypto: ICryptoProvider
log: Logger
@ -49,7 +49,6 @@ export interface NetworkManagerParams {
writerMap: TlWriterMap
isPremium: boolean
_emitError: (err: Error, connection?: SessionConnection) => void
keepAliveAction: () => void
onUsable: () => void
}
@ -238,7 +237,7 @@ export class DcConnectionManager {
/** DC ID */
readonly dcId: number,
/** DC options to use */
readonly _dcs: ITelegramStorage.DcOptions,
readonly _dcs: DcOptions,
/** Whether this DC is the primary one */
public isPrimary = false,
) {
@ -278,7 +277,7 @@ export class DcConnectionManager {
this.upload.setAuthKey(key)
this.download.setAuthKey(key)
this.downloadSmall.setAuthKey(key)
Promise.resolve(this.manager._storage.setAuthKeyFor(this.dcId, key))
Promise.resolve(this.manager._storage.provider.authKeys.set(this.dcId, key))
.then(() => {
this.upload.notifyKeyChange()
this.download.notifyKeyChange()
@ -300,7 +299,7 @@ export class DcConnectionManager {
this.download.setAuthKey(key, true)
this.downloadSmall.setAuthKey(key, true)
Promise.resolve(this.manager._storage.setTempAuthKeyFor(this.dcId, idx, key, expires * 1000))
Promise.resolve(this.manager._storage.provider.authKeys.setTemp(this.dcId, idx, key, expires * 1000))
.then(() => {
this.upload.notifyKeyChange()
this.download.notifyKeyChange()
@ -309,7 +308,7 @@ export class DcConnectionManager {
.catch((e: Error) => this.manager.params._emitError(e))
})
connection.on('future-salts', (salts: mtp.RawMt_future_salt[]) => {
Promise.resolve(this.manager._storage.setFutureSalts(this.dcId, salts)).catch((e: Error) =>
Promise.resolve(this.manager._storage.salts.store(this.dcId, salts)).catch((e: Error) =>
this.manager.params._emitError(e),
)
})
@ -366,8 +365,8 @@ export class DcConnectionManager {
async loadKeys(forcePfs = false): Promise<boolean> {
const [permanent, salts] = await Promise.all([
this.manager._storage.getAuthKeyFor(this.dcId),
this.manager._storage.getFutureSalts(this.dcId),
this.manager._storage.provider.authKeys.get(this.dcId),
this.manager._storage.salts.fetch(this.dcId),
])
this.main.setAuthKey(permanent)
@ -384,9 +383,10 @@ export class DcConnectionManager {
}
if (this.manager.params.usePfs || forcePfs) {
const now = Date.now()
await Promise.all(
this.main._sessions.map(async (_, i) => {
const temp = await this.manager._storage.getAuthKeyFor(this.dcId, i)
const temp = await this.manager._storage.provider.authKeys.getTemp(this.dcId, i, now)
this.main.setAuthKey(temp, true, i)
if (i === 0) {
@ -425,8 +425,6 @@ export class NetworkManager {
protected readonly _dcConnections = new Map<number, DcConnectionManager>()
protected _primaryDc?: DcConnectionManager
private _keepAliveInterval?: NodeJS.Timeout
private _lastUpdateTime = 0
private _updateHandler: (upd: tl.TypeUpdates, fromClient: boolean) => void = () => {}
constructor(
@ -465,7 +463,7 @@ export class NetworkManager {
config.onConfigUpdate(this._onConfigChanged)
}
private async _findDcOptions(dcId: number): Promise<ITelegramStorage.DcOptions> {
private async _findDcOptions(dcId: number): Promise<DcOptions> {
const main = await this.config.findOption({
dcId,
allowIpv6: this.params.useIpv6,
@ -499,21 +497,9 @@ export class NetworkManager {
dc.setIsPrimary(true)
dc.main.on('usable', () => {
this._lastUpdateTime = Date.now()
this.params.onUsable()
if (this._keepAliveInterval) clearInterval(this._keepAliveInterval)
this._keepAliveInterval = setInterval(() => {
if (Date.now() - this._lastUpdateTime > 900_000) {
// telegram asks to fetch pending updates if there are no updates for 15 minutes.
// it is up to the user to decide whether to do it or not
this.params.keepAliveAction()
this._lastUpdateTime = Date.now()
}
}, 60_000)
Promise.resolve(this._storage.getSelf())
Promise.resolve(this._storage.self.fetch())
.then((self) => {
if (self?.isBot) {
// bots may receive tmpSessions, which we should respect
@ -523,7 +509,6 @@ export class NetworkManager {
.catch((e: Error) => this.params._emitError(e))
})
dc.main.on('update', (update: tl.TypeUpdates) => {
this._lastUpdateTime = Date.now()
this._updateHandler(update, false)
})
@ -569,7 +554,7 @@ export class NetworkManager {
*
* @param defaultDcs Default DCs to connect to
*/
async connect(defaultDcs: ITelegramStorage.DcOptions): Promise<void> {
async connect(defaultDcs: DcOptions): Promise<void> {
if (defaultDcs.main.id !== defaultDcs.media.id) {
throw new MtArgumentError('Default DCs must be the same')
}
@ -668,7 +653,7 @@ export class NetworkManager {
this._dcConnections.set(newDc, new DcConnectionManager(this, newDc, options, true))
}
await this._storage.setDefaultDcs(options)
await this._storage.dcs.store(options)
await this._switchPrimaryDc(this._dcConnections.get(newDc)!)
}
@ -723,10 +708,6 @@ export class NetworkManager {
try {
const res = await multi.sendRpc(message, stack, params?.timeout, params?.abortSignal, params?.chainId)
if (kind === 'main') {
this._lastUpdateTime = Date.now()
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return res
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -843,7 +824,6 @@ export class NetworkManager {
for (const dc of this._dcConnections.values()) {
dc.destroy()
}
if (this._keepAliveInterval) clearInterval(this._keepAliveInterval)
this.config.offConfigUpdate(this._onConfigChanged)
}
}

6
packages/core/src/network/persistent-connection.ts
View file

@ -1,16 +1,14 @@
import EventEmitter from 'events'
import { tl } from '@mtcute/tl'
import { MtcuteError } from '../types/index.js'
import { ICryptoProvider, Logger } from '../utils/index.js'
import { BasicDcOption, ICryptoProvider, Logger } from '../utils/index.js'
import { ReconnectionStrategy } from './reconnection.js'
import { ITelegramTransport, TransportFactory, TransportState } from './transports/index.js'
export interface PersistentConnectionParams {
crypto: ICryptoProvider
transportFactory: TransportFactory
dc: tl.RawDcOption
dc: BasicDcOption
testMode: boolean
reconnectionStrategy: ReconnectionStrategy<PersistentConnectionParams>
inactivityTimeout?: number

1
packages/core/src/network/session-connection.ts
View file

@ -250,6 +250,7 @@ export class SessionConnection extends PersistentConnection {
// we must send some user-related rpc to the server to make sure that
// it will send us updates
this.sendRpc({ _: 'updates.getState' }).catch((err: any) => {
if (this._destroyed) return // silently fail
this.log.warn('failed to send updates.getState: %s', err.text || err.message)
})
}

6
packages/core/src/network/transports/abstract.ts
View file

@ -3,7 +3,7 @@ import EventEmitter from 'events'
import { tl } from '@mtcute/tl'
import { MaybeAsync } from '../../types/index.js'
import { ICryptoProvider, Logger } from '../../utils/index.js'
import { BasicDcOption, ICryptoProvider, Logger } from '../../utils/index.js'
/** Current state of the transport */
export enum TransportState {
@ -40,13 +40,13 @@ export interface ITelegramTransport extends EventEmitter {
/** returns current state */
state(): TransportState
/** returns current DC. should return null if state == IDLE */
currentDc(): tl.RawDcOption | null
currentDc(): BasicDcOption | null
/**
* Start trying to connect to a specified DC.
* Will throw an error if state != IDLE
*/
connect(dc: tl.RawDcOption, testMode: boolean): void
connect(dc: BasicDcOption, testMode: boolean): void
/** call to close existing connection to some DC */
close(): void
/** send a message */

10
packages/core/src/network/transports/tcp.ts
View file

@ -1,10 +1,8 @@
import EventEmitter from 'events'
import { connect, Socket } from 'net'
import { tl } from '@mtcute/tl'
import { MtcuteError } from '../../types/errors.js'
import { ICryptoProvider, Logger } from '../../utils/index.js'
import { BasicDcOption, ICryptoProvider, Logger } from '../../utils/index.js'
import { IPacketCodec, ITelegramTransport, TransportState } from './abstract.js'
import { IntermediatePacketCodec } from './intermediate.js'
@ -13,7 +11,7 @@ import { IntermediatePacketCodec } from './intermediate.js'
* Subclasses must provide packet codec in `_packetCodec` property
*/
export abstract class BaseTcpTransport extends EventEmitter implements ITelegramTransport {
protected _currentDc: tl.RawDcOption | null = null
protected _currentDc: BasicDcOption | null = null
protected _state: TransportState = TransportState.Idle
protected _socket: Socket | null = null
@ -41,12 +39,12 @@ export abstract class BaseTcpTransport extends EventEmitter implements ITelegram
return this._state
}
currentDc(): tl.RawDcOption | null {
currentDc(): BasicDcOption | null {
return this._currentDc
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
connect(dc: tl.RawDcOption, testMode: boolean): void {
connect(dc: BasicDcOption, testMode: boolean): void {
if (this._state !== TransportState.Idle) {
throw new MtcuteError('Transport is not IDLE')
}

10
packages/core/src/network/transports/websocket.ts
View file

@ -1,9 +1,7 @@
import EventEmitter from 'events'
import { tl } from '@mtcute/tl'
import { MtcuteError, MtUnsupportedError } from '../../types/errors.js'
import { ICryptoProvider, Logger } from '../../utils/index.js'
import { BasicDcOption, ICryptoProvider, Logger } from '../../utils/index.js'
import { IPacketCodec, ITelegramTransport, TransportState } from './abstract.js'
import { IntermediatePacketCodec } from './intermediate.js'
import { ObfuscatedPacketCodec } from './obfuscated.js'
@ -25,7 +23,7 @@ const subdomainsMap: Record<string, string> = {
* Subclasses must provide packet codec in `_packetCodec` property
*/
export abstract class BaseWebSocketTransport extends EventEmitter implements ITelegramTransport {
private _currentDc: tl.RawDcOption | null = null
private _currentDc: BasicDcOption | null = null
private _state: TransportState = TransportState.Idle
private _socket: WebSocket | null = null
private _crypto!: ICryptoProvider
@ -87,11 +85,11 @@ export abstract class BaseWebSocketTransport extends EventEmitter implements ITe
return this._state
}
currentDc(): tl.RawDcOption | null {
currentDc(): BasicDcOption | null {
return this._currentDc
}
connect(dc: tl.RawDcOption, testMode: boolean): void {
connect(dc: BasicDcOption, testMode: boolean): void {
if (this._state !== TransportState.Idle) {
throw new MtcuteError('Transport is not IDLE')
}

247
packages/core/src/storage/abstract.ts
View file

@ -1,247 +0,0 @@
import { mtp, tl } from '@mtcute/tl'
import { TlReaderMap, TlWriterMap } from '@mtcute/tl-runtime'
import { BasicPeerType, MaybeAsync } from '../types/index.js'
import { Logger } from '../utils/index.js'
// eslint-disable-next-line @typescript-eslint/no-namespace
export namespace ITelegramStorage {
/** Information about a cached peer */
export interface PeerInfo {
/** Peer marked ID */
id: number
/** Peer access hash */
accessHash: tl.Long
/** Peer type */
type: BasicPeerType
/** Peer username, if any */
username?: string
/** Peer phone number, if available */
phone?: string
/** Full TL object with the cached entity */
full: tl.TypeUser | tl.TypeChat
}
/** Information about currently logged in user */
export interface SelfInfo {
/** Whether this is a bot */
isBot: boolean
/** Current user's ID */
userId: number
}
/** Information about preferred DC-s for the user */
export interface DcOptions {
/** Main DC */
main: tl.RawDcOption
/** Media DC. Can be the same as main */
media: tl.RawDcOption
}
}
/**
* Abstract interface for persistent storage.
*
* In some cases you may want to extend existing MemorySession
* and override save()/load()/destroy() methods, but you are also free
* to implement your own session (be sure to refer to MemorySession
* source code to avoid shooting your leg though)
*
* Note that even though set methods *can* be async, you should only
* write updates to the disk when `save()` is called.
*/
export interface ITelegramStorage {
/**
* This method is called before any other.
* For storages that use logging, logger instance.
* For storages that use binary storage, binary maps
*/
setup?(log: Logger, readerMap: TlReaderMap, writerMap: TlWriterMap): void
/**
* Load session from some external storage.
* Should be used either to load session content from file/network/etc
* to memory, or to open required connections to fetch session content later
*/
load?(): MaybeAsync<void>
/**
* Save session to some external storage.
* Should be used to commit pending changes in the session.
* For example, saving session content to file/network/etc,
* or committing a database transaction
*/
save?(): MaybeAsync<void>
/**
* Cleanup session and release all used resources.
*/
destroy?(): MaybeAsync<void>
/**
* Reset session to its default state, optionally resetting auth keys
*
* @param [withAuthKeys=false] Whether to also reset auth keys
*/
reset(withAuthKeys?: boolean): MaybeAsync<void>
/**
* Set default datacenter to use with this session.
*/
setDefaultDcs(dcs: ITelegramStorage.DcOptions | null): MaybeAsync<void>
/**
* Get default datacenter for this session
* (by default should return null)
*/
getDefaultDcs(): MaybeAsync<ITelegramStorage.DcOptions | null>
/**
* Store information about future salts for a given DC
*/
setFutureSalts(dcId: number, salts: mtp.RawMt_future_salt[]): MaybeAsync<void>
/**
* Get information about future salts for a given DC (if available)
*
* You don't need to implement any checks, they will be done by the library.
* It is enough to just return the same array that was passed to `setFutureSalts`.
*/
getFutureSalts(dcId: number): MaybeAsync<mtp.RawMt_future_salt[] | null>
/**
* Get auth_key for a given DC
* (returning null will start authorization)
* For temp keys: should also return null if the key has expired
*
* @param dcId DC ID
* @param tempIndex Index of the temporary key (usually 0, used for multi-connections)
*/
getAuthKeyFor(dcId: number, tempIndex?: number): MaybeAsync<Uint8Array | null>
/**
* Set auth_key for a given DC
*/
setAuthKeyFor(dcId: number, key: Uint8Array | null): MaybeAsync<void>
/**
* Set temp_auth_key for a given DC
* expiresAt is unix time in ms
*/
setTempAuthKeyFor(dcId: number, index: number, key: Uint8Array | null, expiresAt: number): MaybeAsync<void>
/**
* Remove all saved auth keys (both temp and perm)
* for the given DC. Used when perm_key becomes invalid,
* meaning all temp_keys also become invalid
*/
dropAuthKeysFor(dcId: number): MaybeAsync<void>
/**
* Get information about currently logged in user (if available)
*/
getSelf(): MaybeAsync<ITelegramStorage.SelfInfo | null>
/**
* Save information about currently logged in user
*/
setSelf(self: ITelegramStorage.SelfInfo | null): MaybeAsync<void>
/**
* Update local database of input peers from the peer info list
*
* Client will call `.save()` after all updates-related methods
* are called, so you can safely batch these updates
*/
updatePeers(peers: ITelegramStorage.PeerInfo[]): MaybeAsync<void>
/**
* Find a peer in local database by its marked ID
*
* If no peer was found, the storage should try searching its
* reference messages database. If a reference message is found,
* a `inputPeer*FromMessage` constructor should be returned
*/
getPeerById(peerId: number): MaybeAsync<tl.TypeInputPeer | null>
/**
* Find a peer in local database by its username
*/
getPeerByUsername(username: string): MaybeAsync<tl.TypeInputPeer | null>
/**
* Find a peer in local database by its phone number
*/
getPeerByPhone(phone: string): MaybeAsync<tl.TypeInputPeer | null>
/**
* For `*FromMessage` constructors: store a reference to a `peerId` -
* it was seen in message `messageId` in chat `chatId`.
*
* `peerId` and `chatId` are marked peer IDs.
*
* Learn more: https://core.telegram.org/api/min
*/
saveReferenceMessage(peerId: number, chatId: number, messageId: number): MaybeAsync<void>
/**
* For `*FromMessage` constructors: messages `messageIds` in chat `chatId` were deleted,
* so remove any stored peer references to them.
*/
deleteReferenceMessages(chatId: number, messageIds: number[]): MaybeAsync<void>
/**
* Get updates state (if available), represented as a tuple
* containing: `pts, qts, date, seq`
*/
getUpdatesState(): MaybeAsync<[number, number, number, number] | null>
/**
* Set common `pts` value
*
* Client will call `.save()` after all updates-related methods
* are called, so you can safely batch these updates
*/
setUpdatesPts(val: number): MaybeAsync<void>
/**
* Set common `qts` value
*
* Client will call `.save()` after all updates-related methods
* are called, so you can safely batch these updates
*/
setUpdatesQts(val: number): MaybeAsync<void>
/**
* Set updates `date` value
*
* Client will call `.save()` after all updates-related methods
* are called, so you can safely batch these updates
*/
setUpdatesDate(val: number): MaybeAsync<void>
/**
* Set updates `seq` value
*
* Client will call `.save()` after all updates-related methods
* are called, so you can safely batch these updates
*/
setUpdatesSeq(val: number): MaybeAsync<void>
/**
* Get channel `pts` value
*/
getChannelPts(entityId: number): MaybeAsync<number | null>
/**
* Set channels `pts` values in batch.
*
* Storage is supposed to replace stored channel `pts` values
* with given in the object (key is unmarked peer id, value is the `pts`)
*
* Client will call `.save()` after all updates-related methods
* are called, so you can safely batch these updates
*/
setManyChannelPts(values: Map<number, number>): MaybeAsync<void>
/**
* Get cached peer information by their marked ID.
* Return `null` if caching is not supported, or the entity
* is not cached (yet).
*
* This is primarily used when a `min` entity is encountered
* in an update, or when a *short* update is encountered.
* Returning `null` will require re-fetching that
* update with the peers added, which might not be very efficient.
*/
getFullPeerById(id: number): MaybeAsync<tl.TypeUser | tl.TypeChat | null>
}

90
packages/core/src/storage/driver.ts Normal file
View file

@ -0,0 +1,90 @@
import { MaybeAsync } from '../types/utils.js'
import { Logger } from '../utils/logger.js'
/**
* Basic storage driver interface,
* describing the lifecycle of a storage driver
*/
export interface IStorageDriver {
/**
* Load session from some external storage.
* Should be used either to load data from file/network/etc
* to memory, or to open required connections to fetch data on demand
*
* May be called more than once, handle this with care
* (or use {@link BaseStorageDriver} that handles this for you)
*/
load?(): MaybeAsync<void>
/**
* Save session to some external storage.
* Should be used to commit pending changes in the session.
* For example, saving session content to file/network/etc,
* or committing a database transaction
*
* It is safe to batch all changes and only commit them here,
* unless stated otherwise in the method description
*/
save?(): MaybeAsync<void>
/**
* Cleanup session and release all used resources.
*
* May be called more than once, handle this with care
* (or use {@link BaseStorageDriver} that handles this for you)
*/
destroy?(): MaybeAsync<void>
/**
* Setup the driver, passing the logger instance,
* in case your driver needs it
*/
setup?(log: Logger): void
}
/**
* Base storage driver class, implementing {@link IStorageDriver}
* and handling the lifecycle for you
*/
export abstract class BaseStorageDriver implements IStorageDriver {
abstract _load(): MaybeAsync<void>
abstract _destroy(): MaybeAsync<void>
abstract _save?(): MaybeAsync<void>
private _loadedTimes = 0
private _destroyed = false
protected _log!: Logger
setup(log: Logger): void {
this._log = log
}
protected get loaded(): boolean {
return this._loadedTimes > 0
}
async load(): Promise<void> {
if (this._loadedTimes === 0) {
await this._load()
this._destroyed = false
}
this._loadedTimes++
}
async destroy(): Promise<void> {
if (this._destroyed) {
return
}
this._loadedTimes--
if (this._loadedTimes === 0) {
await this._destroy()
this._destroyed = true
}
}
save(): MaybeAsync<void> {
return this._save?.()
}
}

24
packages/core/src/storage/idb.test.ts
View file

@ -1,24 +0,0 @@
import { afterAll, describe } from 'vitest'
import { testStateStorage, testStorage } from '@mtcute/test'
import { IdbStorage } from './idb.js'
describe.skipIf(import.meta.env.TEST_ENV !== 'browser')('IdbStorage', () => {
const idbName = 'mtcute_test_' + Math.random().toString(36).slice(2)
const storage = new IdbStorage(idbName)
testStorage(storage)
testStateStorage(storage)
afterAll(async () => {
storage.destroy()
const req = indexedDB.deleteDatabase(idbName)
await new Promise<void>((resolve, reject) => {
req.onerror = () => reject(req.error)
req.onsuccess = () => resolve()
req.onblocked = () => resolve()
})
})
})

685
packages/core/src/storage/idb.ts
View file

@ -1,685 +0,0 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
import { mtp, tl } from '@mtcute/tl'
import { TlBinaryReader, TlBinaryWriter, TlReaderMap, TlWriterMap } from '@mtcute/tl-runtime'
import { Logger } from '../utils/logger.js'
import { longFromFastString, longToFastString } from '../utils/long-utils.js'
import { LruMap } from '../utils/lru-map.js'
import { toggleChannelIdMark } from '../utils/peer-utils.js'
import { ITelegramStorage } from './abstract.js'
const CURRENT_VERSION = 1
const TABLES = {
kv: 'kv',
state: 'state',
authKeys: 'auth_keys',
tempAuthKeys: 'temp_auth_keys',
pts: 'pts',
entities: 'entities',
messageRefs: 'message_refs',
} as const
const EMPTY_BUFFER = new Uint8Array(0)
interface AuthKeyDto {
dc: number
key: Uint8Array
expiresAt?: number
}
interface EntityDto {
id: number
hash: string
type: string
username?: string
phone?: string
updated: number
full: Uint8Array
}
interface MessageRefDto {
peerId: number
chatId: number
msgId: number
}
interface FsmItemDto {
key: string
value: string
expires?: number
}
function txToPromise(tx: IDBTransaction): Promise<void> {
return new Promise((resolve, reject) => {
tx.oncomplete = () => resolve()
tx.onerror = () => reject(tx.error)
})
}
function reqToPromise<T>(req: IDBRequest<T>): Promise<T> {
return new Promise((resolve, reject) => {
req.onsuccess = () => resolve(req.result)
req.onerror = () => reject(req.error)
})
}
function getInputPeer(row: EntityDto | ITelegramStorage.PeerInfo): tl.TypeInputPeer {
const id = row.id
switch (row.type) {
case 'user':
return {
_: 'inputPeerUser',
userId: id,
accessHash: 'accessHash' in row ? row.accessHash : longFromFastString(row.hash),
}
case 'chat':
return {
_: 'inputPeerChat',
chatId: -id,
}
case 'channel':
return {
_: 'inputPeerChannel',
channelId: toggleChannelIdMark(id),
accessHash: 'accessHash' in row ? row.accessHash : longFromFastString(row.hash),
}
}
throw new Error(`Invalid peer type: ${row.type}`)
}
interface CachedEntity {
peer: tl.TypeInputPeer
full: tl.TypeUser | tl.TypeChat | null
}
/**
* mtcute storage that uses IndexedDB as a backend.
*
* This storage is the default one for browsers, and is generally
* recommended over local storage based one.
*/
export class IdbStorage implements ITelegramStorage {
private _cache?: LruMap<number, CachedEntity>
private _vacuumTimeout?: NodeJS.Timeout
private _vacuumInterval: number
constructor(
readonly _dbName: string,
params?: {
/**
* Entities cache size, in number of entities.
*
* Recently encountered entities are cached in memory,
* to avoid redundant database calls. Set to 0 to
* disable caching (not recommended)
*
* Note that by design in-memory cached is only
* used when finding peer by ID, since other
* kinds of lookups (phone, username) may get stale quickly
*
* @default `100`
*/
cacheSize?: number
/**
* Updates to already cached in-memory entities are only
* applied in DB once in a while, to avoid redundant
* DB calls.
*
* If you are having issues with this, you can set this to `0`
*
* @default `30000` (30 sec)
*/
unimportantSavesDelay?: number
/**
* Interval in milliseconds for vacuuming the storage.
*
* When vacuuming, the storage will remove expired FSM
* states to reduce disk and memory usage.
*
* @default `300_000` (5 minutes)
*/
vacuumInterval?: number
},
) {
if (params?.cacheSize !== 0) {
this._cache = new LruMap(params?.cacheSize ?? 100)
}
this._vacuumInterval = params?.vacuumInterval ?? 300_000
}
db!: IDBDatabase
private _upgradeDb(db: IDBDatabase, oldVer: number, newVer: number): void {
while (oldVer < newVer) {
switch (oldVer) {
case 0: {
db.createObjectStore(TABLES.kv, { keyPath: 'key' })
db.createObjectStore(TABLES.authKeys, { keyPath: 'dc' })
db.createObjectStore(TABLES.tempAuthKeys, { keyPath: ['dc', 'idx'] })
db.createObjectStore(TABLES.pts, { keyPath: 'channelId' })
const stateOs = db.createObjectStore(TABLES.state, { keyPath: 'key' })
stateOs.createIndex('by_expires', 'expires')
const entitiesOs = db.createObjectStore(TABLES.entities, { keyPath: 'id' })
entitiesOs.createIndex('by_username', 'username')
entitiesOs.createIndex('by_phone', 'phone')
const msgRefsOs = db.createObjectStore(TABLES.messageRefs, { keyPath: 'peerId' })
msgRefsOs.createIndex('by_msg', ['chatId', 'msgId'])
oldVer++
}
}
}
if (newVer !== CURRENT_VERSION) throw new Error(`Invalid db version: ${newVer}`)
}
private log!: Logger
private readerMap!: TlReaderMap
private writerMap!: TlWriterMap
private _reader!: TlBinaryReader
setup(log: Logger, readerMap: TlReaderMap, writerMap: TlWriterMap): void {
this.log = log.create('idb')
this.readerMap = readerMap
this.writerMap = writerMap
this._reader = new TlBinaryReader(readerMap, EMPTY_BUFFER)
}
private _pendingWrites: [string, unknown][] = []
private _pendingWritesOses = new Set<string>()
private _writeLater(table: string, obj: unknown): void {
this._pendingWrites.push([table, obj])
this._pendingWritesOses.add(table)
}
private _readFullPeer(data: Uint8Array): tl.TypeUser | tl.TypeChat | null {
this._reader = new TlBinaryReader(this.readerMap, data)
let obj
try {
obj = this._reader.object()
} catch (e) {
// object might be from an older tl layer, in which case it will be ignored
obj = null
}
return obj as tl.TypeUser | tl.TypeChat | null
}
async load(): Promise<void> {
this.db = await new Promise((resolve, reject) => {
const req = indexedDB.open(this._dbName, CURRENT_VERSION)
req.onerror = () => reject(req.error)
req.onsuccess = () => resolve(req.result)
req.onupgradeneeded = (event) =>
this._upgradeDb(req.result, event.oldVersion, event.newVersion || CURRENT_VERSION)
})
this._vacuumTimeout = setInterval(() => {
this._vacuum().catch((e) => {
this.log.warn('Failed to vacuum database: %s', e)
})
}, this._vacuumInterval)
}
async save() {
if (this._pendingWritesOses.size === 0) return
const writes = this._pendingWrites
const oses = this._pendingWritesOses
this._pendingWrites = []
this._pendingWritesOses = new Set()
const tx = this.db.transaction(oses, 'readwrite')
const osMap = new Map<string, IDBObjectStore>()
for (const table of oses) {
osMap.set(table, tx.objectStore(table))
}
for (const [table, obj] of writes) {
const os = osMap.get(table)!
if (obj === null) {
os.delete(table)
} else {
os.put(obj)
}
}
await txToPromise(tx)
}
private async _vacuum(): Promise<void> {
const tx = this.db.transaction(TABLES.state, 'readwrite')
const os = tx.objectStore(TABLES.state)
const keys = await reqToPromise(os.index('by_expires').getAllKeys(IDBKeyRange.upperBound(Date.now())))
for (const key of keys) {
os.delete(key)
}
return txToPromise(tx)
}
destroy(): void {
this.db.close()
clearInterval(this._vacuumTimeout)
}
async reset(withAuthKeys?: boolean | undefined): Promise<void> {
this._cache?.clear()
const tx = this.db.transaction(Object.values(TABLES), 'readwrite')
for (const table of Object.values(TABLES)) {
if (table === TABLES.authKeys && !withAuthKeys) continue
if (table === TABLES.tempAuthKeys && !withAuthKeys) continue
tx.objectStore(table).clear()
}
return txToPromise(tx)
}
private async _getFromKv<T>(key: string): Promise<T | null> {
const tx = this.db.transaction(TABLES.kv)
const store = tx.objectStore(TABLES.kv)
const res = await reqToPromise<{ value: string }>(store.get(key))
if (res === undefined) return null
return JSON.parse(res.value) as T
}
private async _setToKv<T>(key: string, value: T, now = false): Promise<void> {
const dto = { key, value: JSON.stringify(value) }
if (!now) {
this._writeLater(TABLES.kv, dto)
return
}
const tx = this.db.transaction(TABLES.kv, 'readwrite')
const store = tx.objectStore(TABLES.kv)
await reqToPromise(store.put(dto))
}
setDefaultDcs(dcs: ITelegramStorage.DcOptions | null): Promise<void> {
return this._setToKv('dcs', dcs, true)
}
getDefaultDcs(): Promise<ITelegramStorage.DcOptions | null> {
return this._getFromKv('dcs')
}
async getFutureSalts(dcId: number): Promise<mtp.RawMt_future_salt[] | null> {
const res = await this._getFromKv<string[]>(`futureSalts:${dcId}`)
if (!res) return null
return res.map((it) => {
const [salt, validSince, validUntil] = it.split(',')
return {
_: 'mt_future_salt',
validSince: Number(validSince),
validUntil: Number(validUntil),
salt: longFromFastString(salt),
}
})
}
setFutureSalts(dcId: number, salts: mtp.RawMt_future_salt[]): Promise<void> {
return this._setToKv(
`futureSalts:${dcId}`,
salts.map((salt) => `${longToFastString(salt.salt)},${salt.validSince},${salt.validUntil}`),
true,
)
}
async getAuthKeyFor(dcId: number, tempIndex?: number | undefined): Promise<Uint8Array | null> {
let row: AuthKeyDto
if (tempIndex !== undefined) {
const os = this.db.transaction(TABLES.tempAuthKeys).objectStore(TABLES.tempAuthKeys)
row = await reqToPromise<AuthKeyDto>(os.get([dcId, tempIndex]))
if (row === undefined || row.expiresAt! < Date.now()) return null
} else {
const os = this.db.transaction(TABLES.authKeys).objectStore(TABLES.authKeys)
row = await reqToPromise<AuthKeyDto>(os.get(dcId))
if (row === undefined) return null
}
return row.key
}
async setAuthKeyFor(dcId: number, key: Uint8Array | null): Promise<void> {
const os = this.db.transaction(TABLES.authKeys, 'readwrite').objectStore(TABLES.authKeys)
if (key === null) {
return reqToPromise(os.delete(dcId))
}
await reqToPromise(os.put({ dc: dcId, key }))
}
async setTempAuthKeyFor(dcId: number, index: number, key: Uint8Array | null, expiresAt: number): Promise<void> {
const os = this.db.transaction(TABLES.tempAuthKeys, 'readwrite').objectStore(TABLES.tempAuthKeys)
if (key === null) {
return reqToPromise(os.delete([dcId, index]))
}
await reqToPromise(os.put({ dc: dcId, idx: index, key, expiresAt }))
}
async dropAuthKeysFor(dcId: number): Promise<void> {
const tx = this.db.transaction([TABLES.authKeys, TABLES.tempAuthKeys], 'readwrite')
tx.objectStore(TABLES.authKeys).delete(dcId)
// IndexedDB sucks
const tempOs = tx.objectStore(TABLES.tempAuthKeys)
const keys = await reqToPromise(tempOs.getAllKeys())
for (const key of keys) {
if ((key as [number, number])[0] === dcId) {
tempOs.delete(key)
}
}
}
private _cachedSelf?: ITelegramStorage.SelfInfo | null
async getSelf(): Promise<ITelegramStorage.SelfInfo | null> {
if (this._cachedSelf !== undefined) return this._cachedSelf
const self = await this._getFromKv<ITelegramStorage.SelfInfo>('self')
this._cachedSelf = self
return self
}
async setSelf(self: ITelegramStorage.SelfInfo | null): Promise<void> {
this._cachedSelf = self
return this._setToKv('self', self, true)
}
async getUpdatesState(): Promise<[number, number, number, number] | null> {
const os = this.db.transaction(TABLES.kv).objectStore(TABLES.kv)
const [pts, qts, date, seq] = await Promise.all([
reqToPromise<{ value: number }>(os.get('pts')),
reqToPromise<{ value: number }>(os.get('qts')),
reqToPromise<{ value: number }>(os.get('date')),
reqToPromise<{ value: number }>(os.get('seq')),
])
if (pts === undefined || qts === undefined || date === undefined || seq === undefined) return null
return [Number(pts.value), Number(qts.value), Number(date.value), Number(seq.value)]
}
setUpdatesPts(val: number): Promise<void> {
return this._setToKv('pts', val)
}
setUpdatesQts(val: number): Promise<void> {
return this._setToKv('qts', val)
}
setUpdatesDate(val: number): Promise<void> {
return this._setToKv('date', val)
}
setUpdatesSeq(val: number): Promise<void> {
return this._setToKv('seq', val)
}
async getChannelPts(entityId: number): Promise<number | null> {
const os = this.db.transaction(TABLES.pts).objectStore(TABLES.pts)
const row = await reqToPromise<{ pts: number }>(os.get(entityId))
if (row === undefined) return null
return row.pts
}
async setManyChannelPts(values: Map<number, number>): Promise<void> {
const tx = this.db.transaction(TABLES.pts, 'readwrite')
const os = tx.objectStore(TABLES.pts)
for (const [id, pts] of values) {
os.put({ channelId: id, pts })
}
return txToPromise(tx)
}
updatePeers(peers: ITelegramStorage.PeerInfo[]): void {
for (const peer of peers) {
const dto: EntityDto = {
id: peer.id,
hash: longToFastString(peer.accessHash),
type: peer.type,
username: peer.username,
phone: peer.phone,
updated: Date.now(),
full: TlBinaryWriter.serializeObject(this.writerMap, peer.full),
}
this._writeLater(TABLES.entities, dto)
if (!this._cachedSelf?.isBot) {
this._writeLater(TABLES.messageRefs, null)
}
this._cache?.set(peer.id, {
peer: getInputPeer(peer),
full: peer.full,
})
}
}
private async _findPeerByReference(os: IDBObjectStore, peerId: number): Promise<tl.TypeInputPeer | null> {
const row = await reqToPromise<MessageRefDto>(os.get(peerId))
if (row === undefined) return null
const chat = await this.getPeerById(row.chatId, false)
if (chat === null) return null
if (peerId > 0) {
return {
_: 'inputPeerUserFromMessage',
userId: peerId,
peer: chat,
msgId: row.msgId,
}
}
return {
_: 'inputPeerChannelFromMessage',
channelId: toggleChannelIdMark(peerId),
peer: chat,
msgId: row.msgId,
}
}
async getPeerById(peerId: number, allowRefs = true): Promise<tl.TypeInputPeer | null> {
const cached = this._cache?.get(peerId)
if (cached) return cached.peer
const tx = this.db.transaction([TABLES.entities, TABLES.messageRefs])
const entOs = tx.objectStore(TABLES.entities)
const row = await reqToPromise<EntityDto>(entOs.get(peerId))
if (row) {
return getInputPeer(row)
}
if (allowRefs) {
return this._findPeerByReference(tx.objectStore(TABLES.messageRefs), peerId)
}
return null
}
async getPeerByUsername(username: string): Promise<tl.TypeInputPeer | null> {
const tx = this.db.transaction(TABLES.entities)
const os = tx.objectStore(TABLES.entities)
const row = await reqToPromise<EntityDto>(os.index('by_username').get(username))
if (row === undefined) return null
return getInputPeer(row)
}
async getPeerByPhone(phone: string): Promise<tl.TypeInputPeer | null> {
const tx = this.db.transaction(TABLES.entities)
const os = tx.objectStore(TABLES.entities)
const row = await reqToPromise<EntityDto>(os.index('by_phone').get(phone))
if (row === undefined) return null
return getInputPeer(row)
}
async getFullPeerById(peerId: number): Promise<tl.TypeUser | tl.TypeChat | null> {
const cached = this._cache?.get(peerId)
if (cached) return cached.full
const tx = this.db.transaction(TABLES.entities)
const os = tx.objectStore(TABLES.entities)
const row = await reqToPromise<EntityDto>(os.get(peerId))
if (row === undefined) return null
return this._readFullPeer(row.full)
}
async saveReferenceMessage(peerId: number, chatId: number, messageId: number): Promise<void> {
const os = this.db.transaction(TABLES.messageRefs, 'readwrite').objectStore(TABLES.messageRefs)
await reqToPromise(os.put({ peerId, chatId, msgId: messageId } satisfies MessageRefDto))
}
async deleteReferenceMessages(chatId: number, messageIds: number[]): Promise<void> {
const tx = this.db.transaction(TABLES.messageRefs, 'readwrite')
const os = tx.objectStore(TABLES.messageRefs)
const index = os.index('by_msg')
for (const msgId of messageIds) {
const key = await reqToPromise(index.getKey([chatId, msgId]))
if (key === undefined) continue
os.delete(key)
}
return txToPromise(tx)
}
// IStateStorage implementation
async getState(key: string): Promise<unknown> {
const tx = this.db.transaction(TABLES.state, 'readwrite')
const os = tx.objectStore(TABLES.state)
const row = await reqToPromise<FsmItemDto>(os.get(key))
if (!row) return null
if (row.expires && row.expires < Date.now()) {
await reqToPromise(os.delete(key))
return null
}
return JSON.parse(row.value) as unknown
}
async setState(key: string, state: unknown, ttl?: number): Promise<void> {
const tx = this.db.transaction(TABLES.state, 'readwrite')
const os = tx.objectStore(TABLES.state)
const dto: FsmItemDto = {
key,
value: JSON.stringify(state),
expires: ttl ? Date.now() + ttl * 1000 : undefined,
}
await reqToPromise(os.put(dto))
}
async deleteState(key: string): Promise<void> {
const tx = this.db.transaction(TABLES.state, 'readwrite')
const os = tx.objectStore(TABLES.state)
await reqToPromise(os.delete(key))
}
getCurrentScene(key: string): Promise<string | null> {
return this.getState(`$current_scene_${key}`) as Promise<string | null>
}
setCurrentScene(key: string, scene: string, ttl?: number): Promise<void> {
return this.setState(`$current_scene_${key}`, scene, ttl)
}
deleteCurrentScene(key: string): Promise<void> {
return this.deleteState(`$current_scene_${key}`)
}
async getRateLimit(key: string, limit: number, window: number): Promise<[number, number]> {
// leaky bucket
const now = Date.now()
const tx = this.db.transaction(TABLES.state, 'readwrite')
const os = tx.objectStore(TABLES.state)
const row = await reqToPromise<FsmItemDto>(os.get(`$rate_limit_${key}`))
if (!row || row.expires! < now) {
// expired or does not exist
const dto: FsmItemDto = {
key: `$rate_limit_${key}`,
value: limit.toString(),
expires: now + window * 1000,
}
await reqToPromise(os.put(dto))
return [limit, dto.expires!]
}
let value = Number(row.value)
if (value > 0) {
value -= 1
row.value = value.toString()
await reqToPromise(os.put(row))
}
return [value, row.expires!]
}
resetRateLimit(key: string): Promise<void> {
return this.deleteState(`$rate_limit_${key}`)
}
}

7
packages/core/src/storage/index.ts
View file

@ -1,2 +1,5 @@
export * from './abstract.js'
export * from './memory.js'
export * from './driver.js'
export * from './provider.js'
export * from './providers/idb/index.js'
export * from './providers/memory/index.js'
export * from './storage.js'

98
packages/core/src/storage/json-file.ts
View file

@ -1,98 +0,0 @@
// eslint-disable-next-line no-restricted-imports
import * as fs from 'fs'
import { beforeExit } from '../utils/index.js'
import { JsonMemoryStorage } from './json.js'
/**
* mtcute storage that stores data in a JSON file.
*
* > **Note**: This storage is **not fully persistent**, meaning that
* > some data *will* be lost on restart, including entities cache,
* > FSM and rate limiter states, because JSON file would be too large otherwise.
* >
* > This storage should only be used for testing purposes,
* > and should not be used in production. Use e.g. `@mtcute/sqlite` instead.
*
* @deprecated
*/
export class JsonFileStorage extends JsonMemoryStorage {
private readonly _filename: string
private readonly _safe: boolean
private readonly _cleanupUnregister?: () => void
constructor(
filename: string,
params?: {
/**
* Whether to save file "safely", meaning that the file will first be saved
* to `${filename}.tmp`, and then renamed to `filename`,
* instead of writing directly to `filename`.
*
* This solves the issue with the storage being saved as
* a blank file because of the app being stopped while
* the storage is being written.
*
* @default `true`
*/
safe?: boolean
/**
* Whether to save file on process exit.
*
* @default `true`
*/
cleanup?: boolean
},
) {
super()
this._filename = filename
this._safe = params?.safe ?? true
if (params?.cleanup !== false) {
this._cleanupUnregister = beforeExit(() => this._onProcessExit())
}
}
async load(): Promise<void> {
try {
this._loadJson(
await new Promise((res, rej) =>
fs.readFile(this._filename, 'utf-8', (err, data) => (err ? rej(err) : res(data))),
),
)
} catch (e) {}
}
save(): Promise<void> {
return new Promise((resolve, reject) => {
fs.writeFile(this._safe ? this._filename + '.tmp' : this._filename, this._saveJson(), (err) => {
if (err) reject(err)
else if (this._safe) {
fs.rename(this._filename + '.tmp', this._filename, (err) => {
if (err && err.code !== 'ENOENT') reject(err)
else resolve()
})
} else resolve()
})
})
}
private _onProcessExit(): void {
// on exit handler must be synchronous, thus we use sync methods here
try {
fs.writeFileSync(this._filename, this._saveJson())
} catch (e) {}
if (this._safe) {
try {
fs.unlinkSync(this._filename + '.tmp')
} catch (e) {}
}
}
destroy(): void {
this._cleanupUnregister?.()
}
}

42
packages/core/src/storage/json.test.ts
View file

@ -1,42 +0,0 @@
import { describe, expect, it } from 'vitest'
import { JsonMemoryStorage } from './json.js'
// eslint-disable-next-line no-restricted-globals
const createBuffer = import.meta.env.TEST_ENV === 'node' ? Buffer.from : (d: number[]) => new Uint8Array(d)
describe('JsonMemoryStorage', () => {
class ExtJsonMemoryStorage extends JsonMemoryStorage {
loadJson(json: string): void {
this._loadJson(json)
}
saveJson(): string {
return this._saveJson()
}
getInternalState() {
return this._state
}
}
it('should allow importing and exporting to json', () => {
const s = new ExtJsonMemoryStorage()
s.setUpdatesPts(123)
s.setUpdatesQts(456)
// eslint-disable-next-line no-restricted-globals
s.setAuthKeyFor(1, createBuffer([1, 2, 3]))
// eslint-disable-next-line no-restricted-globals
s.setTempAuthKeyFor(2, 0, createBuffer([4, 5, 6]), 1234567890)
const json = s.saveJson()
const s2 = new ExtJsonMemoryStorage()
s2.loadJson(json)
expect(s2.getInternalState()).toEqual({
...s.getInternalState(),
entities: new Map(), // entities are not saved
})
})
})

72
packages/core/src/storage/json.ts
View file

@ -1,72 +0,0 @@
/* eslint-disable @typescript-eslint/no-unsafe-return */
import { base64DecodeToBuffer, base64Encode } from '@mtcute/tl-runtime'
import { MemorySessionState, MemoryStorage } from './memory.js'
/**
* Helper class that provides json serialization functions
* to the session.
*/
export class JsonMemoryStorage extends MemoryStorage {
protected _loadJson(json: string): void {
this._setStateFrom(
JSON.parse(json, (key, value) => {
switch (key) {
case 'authKeys':
case 'authKeysTemp': {
const ret = new Map<string | number, Uint8Array>()
;(value as string).split('|').forEach((pair: string) => {
const [dcId, b64] = pair.split(',')
const mapKey = key === 'authKeysTemp' ? dcId : parseInt(dcId)
ret.set(mapKey, base64DecodeToBuffer(b64))
})
return ret
}
case 'authKeysTempExpiry':
case 'pts':
case 'futureSalts':
return new Map(Object.entries(value as Record<string, string>))
case 'phoneIndex':
case 'usernameIndex':
case 'fsm':
case 'rl':
case 'refs':
case 'entities':
return new Map()
}
return value
}) as MemorySessionState,
)
}
protected _saveJson(): string {
return JSON.stringify(this._state, (key, value) => {
switch (key) {
case 'authKeys':
case 'authKeysTemp': {
const value_ = value as Map<string, Uint8Array | null>
return [...value_.entries()]
.filter((it): it is [string, Uint8Array] => it[1] !== null)
.map(([dcId, key]) => dcId + ',' + base64Encode(key))
.join('|')
}
case 'authKeysTempExpiry':
case 'phoneIndex':
case 'usernameIndex':
case 'pts':
case 'fsm':
case 'rl':
return Object.fromEntries([...(value as Map<string, string>).entries()])
case 'entities':
return {}
}
return value
})
}
}

26
packages/core/src/storage/localstorage.test.ts
View file

@ -1,26 +0,0 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { LocalstorageStorage } from './localstorage.js'
const localStorageStub = {
getItem: vi.fn().mockImplementation(() => null),
setItem: vi.fn(),
}
describe('LocalstorageStorage', () => {
beforeAll(() => void vi.stubGlobal('localStorage', localStorageStub))
afterAll(() => void vi.unstubAllGlobals())
it('should load from localstorage', () => {
const s = new LocalstorageStorage('test')
s.load()
expect(localStorageStub.getItem).toHaveBeenCalledWith('test')
})
it('should save to localstorage', () => {
const s = new LocalstorageStorage('test')
s.save()
expect(localStorageStub.setItem).toHaveBeenCalledWith('test', expect.any(String))
})
})

41
packages/core/src/storage/localstorage.ts
View file

@ -1,41 +0,0 @@
import { MtUnsupportedError } from '../types/index.js'
import { JsonMemoryStorage } from './json.js'
/**
* mtcute storage that stores data in a `localStorage` key.
*
* > **Note**: This storage is **not fully persistent**, meaning that
* > some data *will* be lost on restart, including entities cache,
* > FSM and rate limiter states, because the JSON would be too large otherwise.
* >
* > This storage should only be used for testing purposes,
* > and should not be used in production. Use e.g. {@link IdbStorage} instead.
*
* @deprecated
*/
export class LocalstorageStorage extends JsonMemoryStorage {
private readonly _key: string
constructor(key: string) {
super()
if (typeof localStorage === 'undefined') {
throw new MtUnsupportedError('localStorage is not available!')
}
this._key = key
}
load(): void {
try {
const val = localStorage.getItem(this._key)
if (val === null) return
this._loadJson(val)
} catch (e) {}
}
save(): void {
localStorage.setItem(this._key, this._saveJson())
}
}

55
packages/core/src/storage/memory.test.ts
View file

@ -1,55 +0,0 @@
import { describe, expect, it } from 'vitest'
import { testStateStorage, testStorage } from '@mtcute/test'
import { MemoryStorage } from './memory.js'
describe('MemoryStorage', () => {
testStorage(new MemoryStorage())
testStateStorage(new MemoryStorage())
describe('extending', () => {
it('should allow populating from an object', () => {
class ExtendedMemoryStorage extends MemoryStorage {
constructor() {
super()
this._setStateFrom({
$version: 3,
defaultDcs: null,
authKeys: new Map(),
authKeysTemp: new Map(),
authKeysTempExpiry: new Map(),
entities: new Map(),
phoneIndex: new Map(),
usernameIndex: new Map(),
gpts: [1, 2, 3, 4],
pts: new Map(),
fsm: new Map(),
rl: new Map(),
refs: new Map(),
self: null,
futureSalts: new Map(),
})
}
}
const s = new ExtendedMemoryStorage()
expect(s.getUpdatesState()).toEqual([1, 2, 3, 4])
})
it('should silently fail if version is wrong', () => {
class ExtendedMemoryStorage extends MemoryStorage {
constructor() {
super()
// eslint-disable-next-line
this._setStateFrom({ $version: 0 } as any)
}
}
const s = new ExtendedMemoryStorage()
expect(s.getUpdatesState()).toEqual(null)
})
})
})

524
packages/core/src/storage/memory.ts
View file

@ -1,524 +0,0 @@
import { mtp, tl } from '@mtcute/tl'
import { LruMap, toggleChannelIdMark } from '../utils/index.js'
import { ITelegramStorage } from './abstract.js'
const CURRENT_VERSION = 3
type PeerInfoWithUpdated = ITelegramStorage.PeerInfo & { updated: number }
export interface MemorySessionState {
// forwards compatibility for persistent storages
$version: typeof CURRENT_VERSION
defaultDcs: ITelegramStorage.DcOptions | null
authKeys: Map<number, Uint8Array>
authKeysTemp: Map<string, Uint8Array>
authKeysTempExpiry: Map<string, number>
// marked peer id -> entity info
entities: Map<number, PeerInfoWithUpdated>
// phone number -> peer id
phoneIndex: Map<string, number>
// username -> peer id
usernameIndex: Map<string, number>
// reference messages. peer id -> `${chat id}:${msg id}][]
refs: Map<number, Set<string>>
// common pts, date, seq, qts
gpts: [number, number, number, number] | null
// channel pts
pts: Map<number, number>
// state for fsm
fsm: Map<
string,
{
// value
v: unknown
// expires
e?: number
}
>
// state for rate limiter
rl: Map<
string,
{
// reset
res: number
// remaining
rem: number
}
>
self: ITelegramStorage.SelfInfo | null
futureSalts: Map<number, mtp.RawMt_future_salt[]>
}
const USERNAME_TTL = 86400000 // 24 hours
/**
* In-memory storage implementation for mtcute.
*
* This storage is **not persistent**, meaning that all data
* **will** be lost on restart. Only use this storage for testing,
* or if you know what you're doing.
*/
export class MemoryStorage implements ITelegramStorage {
protected _state!: MemorySessionState
private _cachedInputPeers: LruMap<number, tl.TypeInputPeer> = new LruMap(100)
private _cachedFull: LruMap<number, tl.TypeUser | tl.TypeChat>
private _vacuumTimeout?: NodeJS.Timeout
private _vacuumInterval: number
constructor(params?: {
/**
* Maximum number of cached full entities.
*
* Note that full entities are **NOT** persisted
* to the disk (in case this storage is backed
* by a local storage), and only available within
* the current runtime.
*
* @default `100`, use `0` to disable
*/
cacheSize?: number
/**
* Interval in milliseconds for vacuuming the storage.
*
* When vacuuming, the storage will remove expired FSM
* states to reduce memory usage.
*
* @default `300_000` (5 minutes)
*/
vacuumInterval?: number
}) {
this.reset(true)
this._cachedFull = new LruMap(params?.cacheSize ?? 100)
this._vacuumInterval = params?.vacuumInterval ?? 300_000
}
load(): void {
this._vacuumTimeout = setInterval(this._vacuum.bind(this), this._vacuumInterval)
}
destroy(): void {
clearInterval(this._vacuumTimeout)
}
reset(withAuthKeys = false): void {
this._state = {
$version: CURRENT_VERSION,
defaultDcs: null,
authKeys: withAuthKeys ? new Map<number, Uint8Array>() : this._state.authKeys,
authKeysTemp: withAuthKeys ? new Map<string, Uint8Array>() : this._state.authKeysTemp,
authKeysTempExpiry: withAuthKeys ? new Map<string, number>() : this._state.authKeysTempExpiry,
entities: new Map(),
phoneIndex: new Map(),
usernameIndex: new Map(),
refs: new Map(),
gpts: null,
pts: new Map(),
fsm: new Map(),
rl: new Map(),
self: null,
futureSalts: new Map(),
}
this._cachedInputPeers?.clear()
this._cachedFull?.clear()
}
/**
* Set a given object as an underlying state.
*
* Note that this object will be used as-is, so if
* you plan on using it somewhere else, be sure to copy it beforehand.
*/
protected _setStateFrom(obj: MemorySessionState): void {
let ver = obj.$version as number
if (ver === 1) {
// v2: introduced message references
obj.refs = new Map()
obj.$version = ver = 2 as any // eslint-disable-line
}
if (ver === 2) {
// v3: introduced future salts
obj.futureSalts = new Map()
obj.$version = ver = 3
}
if (ver !== CURRENT_VERSION) return
// populate indexes if needed
let populate = false
if (!obj.phoneIndex?.size) {
obj.phoneIndex = new Map()
populate = true
}
if (!obj.usernameIndex?.size) {
obj.usernameIndex = new Map()
populate = true
}
if (populate) {
Object.values(obj.entities).forEach((ent: ITelegramStorage.PeerInfo) => {
if (ent.phone) obj.phoneIndex.set(ent.phone, ent.id)
if (ent.username) {
obj.usernameIndex.set(ent.username, ent.id)
}
})
}
this._state = obj
}
private _vacuum(): void {
// remove expired entities from fsm and rate limit storages
const now = Date.now()
// make references in advance to avoid lookups
const state = this._state
const fsm = state.fsm
const rl = state.rl
for (const [key, item] of fsm) {
if (item.e && item.e < now) {
fsm.delete(key)
}
}
for (const [key, item] of rl) {
if (item.res < now) {
rl.delete(key)
}
}
}
getDefaultDcs(): ITelegramStorage.DcOptions | null {
return this._state.defaultDcs
}
setDefaultDcs(dcs: ITelegramStorage.DcOptions | null): void {
this._state.defaultDcs = dcs
}
setFutureSalts(dcId: number, salts: mtp.RawMt_future_salt[]): void {
this._state.futureSalts.set(dcId, salts)
}
getFutureSalts(dcId: number): mtp.RawMt_future_salt[] | null {
return this._state.futureSalts.get(dcId) ?? null
}
setTempAuthKeyFor(dcId: number, index: number, key: Uint8Array | null, expiresAt: number): void {
const k = `${dcId}:${index}`
if (key) {
this._state.authKeysTemp.set(k, key)
this._state.authKeysTempExpiry.set(k, expiresAt)
} else {
this._state.authKeysTemp.delete(k)
this._state.authKeysTempExpiry.delete(k)
}
}
setAuthKeyFor(dcId: number, key: Uint8Array | null): void {
if (key) {
this._state.authKeys.set(dcId, key)
} else {
this._state.authKeys.delete(dcId)
}
}
getAuthKeyFor(dcId: number, tempIndex?: number): Uint8Array | null {
if (tempIndex !== undefined) {
const k = `${dcId}:${tempIndex}`
if (Date.now() > (this._state.authKeysTempExpiry.get(k) ?? 0)) {
return null
}
return this._state.authKeysTemp.get(k) ?? null
}
return this._state.authKeys.get(dcId) ?? null
}
dropAuthKeysFor(dcId: number): void {
this._state.authKeys.delete(dcId)
for (const key of this._state.authKeysTemp.keys()) {
if (key.startsWith(`${dcId}:`)) {
this._state.authKeysTemp.delete(key)
this._state.authKeysTempExpiry.delete(key)
}
}
// future salts are linked to auth keys
this._state.futureSalts.delete(dcId)
}
updatePeers(peers: PeerInfoWithUpdated[]): void {
for (const peer of peers) {
this._cachedFull.set(peer.id, peer.full)
peer.updated = Date.now()
const old = this._state.entities.get(peer.id)
if (old) {
// delete old index entries if needed
if (old.username && peer.username !== old.username) {
this._state.usernameIndex.delete(old.username)
}
if (old.phone && old.phone !== peer.phone) {
this._state.phoneIndex.delete(old.phone)
}
}
if (peer.username) {
this._state.usernameIndex.set(peer.username, peer.id)
}
if (peer.phone) this._state.phoneIndex.set(peer.phone, peer.id)
this._state.entities.set(peer.id, peer)
// no point in storing references anymore, since we have the full peer
if (this._state.refs.has(peer.id)) {
this._state.refs.delete(peer.id)
}
}
}
protected _getInputPeer(peerInfo?: ITelegramStorage.PeerInfo): tl.TypeInputPeer | null {
if (!peerInfo) return null
switch (peerInfo.type) {
case 'user':
return {
_: 'inputPeerUser',
userId: peerInfo.id,
accessHash: peerInfo.accessHash,
}
case 'chat':
return {
_: 'inputPeerChat',
chatId: -peerInfo.id,
}
case 'channel':
return {
_: 'inputPeerChannel',
channelId: toggleChannelIdMark(peerInfo.id),
accessHash: peerInfo.accessHash,
}
}
}
private _findPeerByRef(peerId: number): tl.TypeInputPeer | null {
const refs = this._state.refs.get(peerId)
if (!refs || refs.size === 0) return null
const [ref] = refs.values()
const [chatId, msgId] = ref.split(':').map(Number)
const chatPeer = this._getInputPeer(this._state.entities.get(chatId))
if (!chatPeer) return null
if (peerId > 0) {
// user
return {
_: 'inputPeerUserFromMessage',
msgId,
userId: peerId,
peer: chatPeer,
}
}
// channel
return {
_: 'inputPeerChannelFromMessage',
msgId,
channelId: toggleChannelIdMark(peerId),
peer: chatPeer,
}
}
getPeerById(peerId: number): tl.TypeInputPeer | null {
if (this._cachedInputPeers.has(peerId)) {
return this._cachedInputPeers.get(peerId)!
}
let peer = this._getInputPeer(this._state.entities.get(peerId))
if (!peer) peer = this._findPeerByRef(peerId)
if (peer) this._cachedInputPeers.set(peerId, peer)
return peer
}
getPeerByPhone(phone: string): tl.TypeInputPeer | null {
const peerId = this._state.phoneIndex.get(phone)
if (!peerId) return null
return this._getInputPeer(this._state.entities.get(peerId))
}
getPeerByUsername(username: string): tl.TypeInputPeer | null {
const id = this._state.usernameIndex.get(username.toLowerCase())
if (!id) return null
const peer = this._state.entities.get(id)
if (!peer) return null
if (Date.now() - peer.updated > USERNAME_TTL) return null
return this._getInputPeer(peer)
}
saveReferenceMessage(peerId: number, chatId: number, messageId: number): void {
if (!this._state.refs.has(peerId)) {
this._state.refs.set(peerId, new Set())
}
this._state.refs.get(peerId)!.add(`${chatId}:${messageId}`)
}
deleteReferenceMessages(chatId: number, messageIds: number[]): void {
// not the most efficient way, but it's fine
for (const refs of this._state.refs.values()) {
for (const msg of messageIds) {
refs.delete(`${chatId}:${msg}`)
}
}
}
getSelf(): ITelegramStorage.SelfInfo | null {
return this._state.self
}
setSelf(self: ITelegramStorage.SelfInfo | null): void {
this._state.self = self
}
setManyChannelPts(values: Map<number, number>): void {
for (const [id, pts] of values) {
this._state.pts.set(id, pts)
}
}
getChannelPts(entityId: number): number | null {
return this._state.pts.get(entityId) ?? null
}
getUpdatesState(): [number, number, number, number] | null {
return this._state.gpts ?? null
}
setUpdatesPts(val: number): void {
if (!this._state.gpts) this._state.gpts = [0, 0, 0, 0]
this._state.gpts[0] = val
}
setUpdatesQts(val: number): void {
if (!this._state.gpts) this._state.gpts = [0, 0, 0, 0]
this._state.gpts[1] = val
}
setUpdatesDate(val: number): void {
if (!this._state.gpts) this._state.gpts = [0, 0, 0, 0]
this._state.gpts[2] = val
}
setUpdatesSeq(val: number): void {
if (!this._state.gpts) this._state.gpts = [0, 0, 0, 0]
this._state.gpts[3] = val
}
getFullPeerById(id: number): tl.TypeUser | tl.TypeChat | null {
return this._cachedFull.get(id) ?? null
}
// IStateStorage implementation
getState(key: string): unknown {
const val = this._state.fsm.get(key)
if (!val) return null
if (val.e && val.e < Date.now()) {
// expired
this._state.fsm.delete(key)
return null
}
return val.v
}
setState(key: string, state: unknown, ttl?: number): void {
this._state.fsm.set(key, {
v: state,
e: ttl ? Date.now() + ttl * 1000 : undefined,
})
}
deleteState(key: string): void {
this._state.fsm.delete(key)
}
getCurrentScene(key: string): string | null {
return this.getState(`$current_scene_${key}`) as string | null
}
setCurrentScene(key: string, scene: string, ttl?: number): void {
return this.setState(`$current_scene_${key}`, scene, ttl)
}
deleteCurrentScene(key: string): void {
this._state.fsm.delete(`$current_scene_${key}`)
}
getRateLimit(key: string, limit: number, window: number): [number, number] {
// leaky bucket
const now = Date.now()
const item = this._state.rl.get(key)
if (!item) {
const state = {
res: now + window * 1000,
rem: limit,
}
this._state.rl.set(key, state)
return [state.rem, state.res]
}
if (item.res < now) {
// expired
const state = {
res: now + window * 1000,
rem: limit,
}
this._state.rl.set(key, state)
return [state.rem, state.res]
}
item.rem = item.rem > 0 ? item.rem - 1 : 0
return [item.rem, item.res]
}
resetRateLimit(key: string): void {
this._state.rl.delete(key)
}
}

16
packages/core/src/storage/provider.ts Normal file
View file

@ -0,0 +1,16 @@
import { IStorageDriver } from './driver.js'
import { IAuthKeysRepository } from './repository/auth-keys.js'
import { IKeyValueRepository } from './repository/key-value.js'
import { IPeersRepository } from './repository/peers.js'
import { IReferenceMessagesRepository } from './repository/ref-messages.js'
export type IStorageProvider<T> = T & {
readonly driver: IStorageDriver
}
export type IMtStorageProvider = IStorageProvider<{
readonly kv: IKeyValueRepository
readonly authKeys: IAuthKeysRepository
readonly peers: IPeersRepository
readonly refMessages: IReferenceMessagesRepository
}>

169
packages/core/src/storage/providers/idb/driver.ts Normal file
View file

@ -0,0 +1,169 @@
import { MtUnsupportedError } from '../../../types/errors.js'
import { BaseStorageDriver } from '../../driver.js'
import { txToPromise } from './utils.js'
export type PostMigrationFunction = (db: IDBDatabase) => Promise<void>
type MigrationFunction = (db: IDBDatabase) => void | PostMigrationFunction
const REPO_VERSION_PREFIX = '__version:'
export class IdbStorageDriver extends BaseStorageDriver {
db!: IDBDatabase
constructor(readonly _dbName: string) {
super()
if (typeof indexedDB === 'undefined') {
throw new MtUnsupportedError('IndexedDB is not available')
}
}
private _pendingWrites: [string, unknown][] = []
private _pendingWritesOses = new Set<string>()
private _migrations: Map<string, Map<number, MigrationFunction>> = new Map()
private _maxVersion: Map<string, number> = new Map()
registerMigration(repo: string, version: number, migration: MigrationFunction): void {
if (this.loaded) {
throw new Error('Cannot register migrations after loading')
}
let map = this._migrations.get(repo)
if (!map) {
map = new Map()
this._migrations.set(repo, map)
}
if (map.has(version)) {
throw new Error(`Migration for ${repo} version ${version} is already registered`)
}
map.set(version, migration)
const prevMax = this._maxVersion.get(repo) ?? 0
if (version > prevMax) {
this._maxVersion.set(repo, version)
}
}
writeLater(os: string, obj: unknown): void {
this._pendingWrites.push([os, obj])
this._pendingWritesOses.add(os)
}
async _load(): Promise<void> {
this.db = await new Promise((resolve, reject) => {
// indexed db fucking sucks - we can't create tables once we have loaded
// and making an ever-incrementing version number is pretty hard
// since migrations are added dynamically.
//
// force the database to always emit `upgradeneeded` by passing current time
const req = indexedDB.open(this._dbName, Date.now())
req.onerror = () => reject(req.error)
const postUpgrade: PostMigrationFunction[] = []
req.onsuccess = async () => {
try {
for (const cb of postUpgrade) {
await cb(req.result)
}
resolve(req.result)
} catch (e) {
reject(e)
}
}
req.onupgradeneeded = () => {
// indexed db still fucking sucks. we can't fetch anything from here,
// since migrations must be sync, and any fetch is inherently async
// what we do have, however, is the list of object stores.
// we can abuse them to store the current migrations status as plain strings
const db = req.result
const didUpgrade = new Set<string>()
const doUpgrade = (repo: string, fromVersion: number) => {
const migrations = this._migrations.get(repo)
if (!migrations) return
const targetVer = this._maxVersion.get(repo)!
while (fromVersion < targetVer) {
const nextVersion = fromVersion + 1
const migration = migrations.get(nextVersion)
if (!migration) {
throw new Error(`No migration for ${repo} to version ${nextVersion}`)
}
const result = migration(db)
if (result) {
// guess what? IDB still. fucking. sucks!
// if we want to do something except creating/removing
// databases, we should do this outside of migration
postUpgrade.push(result)
}
fromVersion = nextVersion
}
didUpgrade.add(repo)
db.createObjectStore(`${REPO_VERSION_PREFIX}${repo}:${targetVer}`)
}
for (const key of db.objectStoreNames) {
if (!key.startsWith(REPO_VERSION_PREFIX)) continue
const [, repo, version] = key.split(':')
const currentVer = Number(version)
db.deleteObjectStore(key)
doUpgrade(repo, currentVer)
didUpgrade.add(repo)
}
for (const repo of this._migrations.keys()) {
if (didUpgrade.has(repo)) continue
doUpgrade(repo, 0)
}
}
})
}
async _save() {
if (this._pendingWritesOses.size === 0) return
const writes = this._pendingWrites
const oses = this._pendingWritesOses
this._pendingWrites = []
this._pendingWritesOses = new Set()
const tx = this.db.transaction(oses, 'readwrite')
const osMap = new Map<string, IDBObjectStore>()
for (const table of oses) {
osMap.set(table, tx.objectStore(table))
}
for (const [table, obj] of writes) {
const os = osMap.get(table)!
if (obj === null) {
os.delete(table)
} else {
os.put(obj)
}
}
await txToPromise(tx)
}
_destroy(): void {
this.db.close()
}
}

35
packages/core/src/storage/providers/idb/idb.test.ts Normal file
View file

@ -0,0 +1,35 @@
import { afterAll, beforeAll, describe } from 'vitest'
import { testAuthKeysRepository } from '../../repository/auth-keys.test-utils.js'
import { testKeyValueRepository } from '../../repository/key-value.test-utils.js'
import { testPeersRepository } from '../../repository/peers.test-utils.js'
import { testRefMessagesRepository } from '../../repository/ref-messages.test-utils.js'
import { IdbStorage } from './index.js'
if (import.meta.env.TEST_ENV === 'browser') {
describe('idb storage', () => {
const idbName = 'mtcute_test_' + Math.random().toString(36).slice(2)
const storage = new IdbStorage(idbName)
beforeAll(() => storage.driver.load())
testAuthKeysRepository(storage.authKeys)
testKeyValueRepository(storage.kv, storage.driver)
testPeersRepository(storage.peers, storage.driver)
testRefMessagesRepository(storage.refMessages, storage.driver)
afterAll(async () => {
storage.driver.destroy()
const req = indexedDB.deleteDatabase(idbName)
await new Promise<void>((resolve, reject) => {
req.onerror = () => reject(req.error)
req.onsuccess = () => resolve()
req.onblocked = () => resolve()
})
})
})
} else {
describe.skip('idb storage', () => {})
}

22
packages/core/src/storage/providers/idb/index.ts Normal file
View file

@ -0,0 +1,22 @@
import { IMtStorageProvider } from '../../provider.js'
import { IdbStorageDriver } from './driver.js'
import { IdbAuthKeysRepository } from './repository/auth-keys.js'
import { IdbKvRepository } from './repository/kv.js'
import { IdbPeersRepository } from './repository/peers.js'
import { IdbRefMsgRepository } from './repository/ref-messages.js'
/**
* mtcute storage that uses IndexedDB as a backend.
*
* This storage is the default one for browsers, and is generally
* recommended over local storage based one.
*/
export class IdbStorage implements IMtStorageProvider {
constructor(readonly dbName: string) {}
readonly driver = new IdbStorageDriver(this.dbName)
readonly kv = new IdbKvRepository(this.driver)
readonly authKeys = new IdbAuthKeysRepository(this.driver)
readonly peers = new IdbPeersRepository(this.driver)
readonly refMessages = new IdbRefMsgRepository(this.driver)
}

96
packages/core/src/storage/providers/idb/repository/auth-keys.ts Normal file
View file

@ -0,0 +1,96 @@
import { IAuthKeysRepository } from '../../../repository/auth-keys.js'
import { IdbStorageDriver } from '../driver.js'
import { reqToPromise, txToPromise } from '../utils.js'
const TABLE_AUTH_KEYS = 'authKeys'
const TABLE_TEMP_AUTH_KEYS = 'tempAuthKeys'
interface AuthKeyDto {
dc: number
key: Uint8Array
}
interface TempAuthKeyDto extends AuthKeyDto {
expiresAt?: number
idx?: number
}
export class IdbAuthKeysRepository implements IAuthKeysRepository {
constructor(readonly _driver: IdbStorageDriver) {
_driver.registerMigration(TABLE_AUTH_KEYS, 1, (db) => {
db.createObjectStore(TABLE_AUTH_KEYS, { keyPath: 'dc' })
db.createObjectStore(TABLE_TEMP_AUTH_KEYS, { keyPath: ['dc', 'idx'] })
})
}
private os(mode?: IDBTransactionMode): IDBObjectStore {
return this._driver.db.transaction(TABLE_AUTH_KEYS, mode).objectStore(TABLE_AUTH_KEYS)
}
async set(dc: number, key: Uint8Array | null): Promise<void> {
const os = this.os('readwrite')
if (key === null) {
return reqToPromise(os.delete(dc))
}
await reqToPromise(os.put({ dc, key } satisfies AuthKeyDto))
}
async get(dc: number): Promise<Uint8Array | null> {
const os = this.os()
const it = await reqToPromise<AuthKeyDto>(os.get(dc))
if (it === undefined) return null
return it.key
}
private osTemp(mode?: IDBTransactionMode): IDBObjectStore {
return this._driver.db.transaction(TABLE_TEMP_AUTH_KEYS, mode).objectStore(TABLE_TEMP_AUTH_KEYS)
}
async setTemp(dc: number, idx: number, key: Uint8Array | null, expires: number): Promise<void> {
const os = this.osTemp('readwrite')
if (!key) {
return reqToPromise(os.delete([dc, idx]))
}
await reqToPromise(os.put({ dc, idx, key, expiresAt: expires } satisfies TempAuthKeyDto))
}
async getTemp(dc: number, idx: number, now: number): Promise<Uint8Array | null> {
const os = this.osTemp()
const row = await reqToPromise<TempAuthKeyDto>(os.get([dc, idx]))
if (row === undefined || row.expiresAt! < now) return null
return row.key
}
async deleteByDc(dc: number): Promise<void> {
const tx = this._driver.db.transaction([TABLE_AUTH_KEYS, TABLE_TEMP_AUTH_KEYS], 'readwrite')
tx.objectStore(TABLE_AUTH_KEYS).delete(dc)
// IndexedDB sucks
const tempOs = tx.objectStore(TABLE_TEMP_AUTH_KEYS)
const keys = await reqToPromise(tempOs.getAllKeys())
for (const key of keys) {
if ((key as [number, number])[0] === dc) {
tempOs.delete(key)
}
}
await txToPromise(tx)
}
deleteAll(): Promise<void> {
const tx = this._driver.db.transaction([TABLE_AUTH_KEYS, TABLE_TEMP_AUTH_KEYS], 'readwrite')
tx.objectStore(TABLE_AUTH_KEYS).clear()
tx.objectStore(TABLE_TEMP_AUTH_KEYS).clear()
return txToPromise(tx)
}
}

41
packages/core/src/storage/providers/idb/repository/kv.ts Normal file
View file

@ -0,0 +1,41 @@
import { IKeyValueRepository } from '../../../repository/key-value.js'
import { IdbStorageDriver } from '../driver.js'
import { reqToPromise } from '../utils.js'
const KV_TABLE = 'kv'
interface KeyValueDto {
key: string
value: Uint8Array
}
export class IdbKvRepository implements IKeyValueRepository {
constructor(readonly _driver: IdbStorageDriver) {
_driver.registerMigration(KV_TABLE, 1, (db) => {
db.createObjectStore(KV_TABLE, { keyPath: 'key' })
})
}
set(key: string, value: Uint8Array): void {
this._driver.writeLater(KV_TABLE, { key, value } satisfies KeyValueDto)
}
private os(mode?: IDBTransactionMode): IDBObjectStore {
return this._driver.db.transaction(KV_TABLE, mode).objectStore(KV_TABLE)
}
async get(key: string): Promise<Uint8Array | null> {
const os = this.os()
const res = await reqToPromise<KeyValueDto>(os.get(key))
if (res === undefined) return null
return res.value
}
async delete(key: string): Promise<void> {
await reqToPromise(this.os('readwrite').delete(key))
}
async deleteAll(): Promise<void> {
await reqToPromise(this.os('readwrite').clear())
}
}

45
packages/core/src/storage/providers/idb/repository/peers.ts Normal file
View file

@ -0,0 +1,45 @@
import { IPeersRepository } from '../../../repository/peers.js'
import { IdbStorageDriver } from '../driver.js'
import { reqToPromise } from '../utils.js'
const TABLE = 'peers'
export class IdbPeersRepository implements IPeersRepository {
constructor(readonly _driver: IdbStorageDriver) {
_driver.registerMigration(TABLE, 1, (db) => {
const os = db.createObjectStore(TABLE, { keyPath: 'id' })
os.createIndex('by_username', 'usernames', { unique: true, multiEntry: true })
os.createIndex('by_phone', 'phone', { unique: true })
})
}
store(peer: IPeersRepository.PeerInfo): void {
this._driver.writeLater(TABLE, peer)
}
private os(mode?: IDBTransactionMode): IDBObjectStore {
return this._driver.db.transaction(TABLE, mode).objectStore(TABLE)
}
async getById(id: number): Promise<IPeersRepository.PeerInfo | null> {
const it = await reqToPromise(this.os().get(id))
return it ?? null
}
async getByUsername(username: string): Promise<IPeersRepository.PeerInfo | null> {
const it = await reqToPromise(this.os().index('by_username').get(username))
return it ?? null
}
async getByPhone(phone: string): Promise<IPeersRepository.PeerInfo | null> {
const it = await reqToPromise(this.os().index('by_phone').get(phone))
return it ?? null
}
deleteAll(): Promise<void> {
return reqToPromise(this.os('readwrite').clear())
}
}

74
packages/core/src/storage/providers/idb/repository/ref-messages.ts Normal file
View file

@ -0,0 +1,74 @@
import { IReferenceMessagesRepository } from '../../../repository/ref-messages.js'
import { IdbStorageDriver } from '../driver.js'
import { cursorToIterator, reqToPromise, txToPromise } from '../utils.js'
const TABLE = 'messageRefs'
interface MessageRefDto {
peerId: number
chatId: number
msgId: number
}
export class IdbRefMsgRepository implements IReferenceMessagesRepository {
constructor(readonly _driver: IdbStorageDriver) {
_driver.registerMigration(TABLE, 1, (db) => {
const os = db.createObjectStore(TABLE, { keyPath: ['peerId', 'chatId', 'msgId'] })
os.createIndex('by_peer', 'peerId')
os.createIndex('by_msg', ['chatId', 'msgId'])
})
}
private os(mode?: IDBTransactionMode): IDBObjectStore {
return this._driver.db.transaction(TABLE, mode).objectStore(TABLE)
}
async store(peerId: number, chatId: number, msgId: number): Promise<void> {
const os = this.os('readwrite')
await reqToPromise(os.put({ peerId, chatId, msgId } satisfies MessageRefDto))
}
async getByPeer(peerId: number): Promise<[number, number] | null> {
const os = this.os()
const index = os.index('by_peer')
const it = await reqToPromise<MessageRefDto>(index.get(peerId))
if (!it) return null
return [it.chatId, it.msgId]
}
async delete(chatId: number, msgIds: number[]): Promise<void> {
const tx = this._driver.db.transaction(TABLE, 'readwrite')
const os = tx.objectStore(TABLE)
const index = os.index('by_msg')
for (const msgId of msgIds) {
const keys = await reqToPromise(index.getAllKeys([chatId, msgId]))
// there are never that many keys, so we can avoid using cursor
for (const key of keys) {
os.delete(key)
}
}
return txToPromise(tx)
}
async deleteByPeer(peerId: number): Promise<void> {
const tx = this._driver.db.transaction(TABLE, 'readwrite')
const os = tx.objectStore(TABLE)
const index = os.index('by_peer')
for await (const cursor of cursorToIterator(index.openCursor(peerId))) {
cursor.delete()
}
return txToPromise(tx)
}
async deleteAll(): Promise<void> {
await reqToPromise(this.os('readwrite').clear())
}
}

25
packages/core/src/storage/providers/idb/utils.ts Normal file
View file

@ -0,0 +1,25 @@
export function txToPromise(tx: IDBTransaction): Promise<void> {
return new Promise((resolve, reject) => {
tx.oncomplete = () => resolve()
tx.onerror = () => reject(tx.error)
})
}
export function reqToPromise<T>(req: IDBRequest<T>): Promise<T> {
return new Promise((resolve, reject) => {
req.onsuccess = () => resolve(req.result)
req.onerror = () => reject(req.error)
})
}
export async function* cursorToIterator<T extends IDBCursor>(
req: IDBRequest<T | null>,
): AsyncIterableIterator<T> {
let cursor = await reqToPromise(req)
while (cursor) {
yield cursor
cursor.continue()
cursor = await reqToPromise(req)
}
}

15
packages/core/src/storage/providers/memory/driver.ts Normal file
View file

@ -0,0 +1,15 @@
import { IStorageDriver } from '../../driver.js'
export class MemoryStorageDriver implements IStorageDriver {
readonly states: Map<string, object> = new Map()
getState<T extends object>(repo: string, def: T) {
if (!this.states.has(repo)) {
this.states.set(repo, def)
}
return this.states.get(repo) as T
}
load() {}
}

21
packages/core/src/storage/providers/memory/index.ts Normal file
View file

@ -0,0 +1,21 @@
import { IMtStorageProvider } from '../../provider.js'
import { MemoryStorageDriver } from './driver.js'
import { MemoryAuthKeysRepository } from './repository/auth-keys.js'
import { MemoryKeyValueRepository } from './repository/kv.js'
import { MemoryPeersRepository } from './repository/peers.js'
import { MemoryRefMessagesRepository } from './repository/ref-messages.js'
/**
* In-memory storage driver implementation for mtcute.
*
* This storage is **not persistent**, meaning that all data
* **will** be lost on restart. Only use this storage for testing,
* or if you know exactly what you're doing.
*/
export class MemoryStorage implements IMtStorageProvider {
readonly driver = new MemoryStorageDriver()
readonly kv = new MemoryKeyValueRepository(this.driver)
readonly authKeys = new MemoryAuthKeysRepository(this.driver)
readonly peers = new MemoryPeersRepository(this.driver)
readonly refMessages = new MemoryRefMessagesRepository(this.driver)
}

16
packages/core/src/storage/providers/memory/memory.test.ts Normal file
View file

@ -0,0 +1,16 @@
import { describe } from 'vitest'
import { testAuthKeysRepository } from '../../repository/auth-keys.test-utils.js'
import { testKeyValueRepository } from '../../repository/key-value.test-utils.js'
import { testPeersRepository } from '../../repository/peers.test-utils.js'
import { testRefMessagesRepository } from '../../repository/ref-messages.test-utils.js'
import { MemoryStorage } from './index.js'
describe('memory storage', () => {
const storage = new MemoryStorage()
testAuthKeysRepository(storage.authKeys)
testKeyValueRepository(storage.kv, storage.driver)
testPeersRepository(storage.peers, storage.driver)
testRefMessagesRepository(storage.refMessages, storage.driver)
})

69
packages/core/src/storage/providers/memory/repository/auth-keys.ts Normal file
View file

@ -0,0 +1,69 @@
import { IAuthKeysRepository } from '../../../repository/auth-keys.js'
import { MemoryStorageDriver } from '../driver.js'
interface AuthKeysState {
authKeys: Map<number, Uint8Array>
authKeysTemp: Map<string, Uint8Array>
authKeysTempExpiry: Map<string, number>
}
export class MemoryAuthKeysRepository implements IAuthKeysRepository {
constructor(readonly _driver: MemoryStorageDriver) {}
readonly state = this._driver.getState<AuthKeysState>('authKeys', {
authKeys: new Map(),
authKeysTemp: new Map(),
authKeysTempExpiry: new Map(),
})
set(dc: number, key: Uint8Array | null): void {
if (key) {
this.state.authKeys.set(dc, key)
} else {
this.state.authKeys.delete(dc)
}
}
get(dc: number): Uint8Array | null {
return this.state.authKeys.get(dc) ?? null
}
setTemp(dc: number, idx: number, key: Uint8Array | null, expires: number): void {
const k = `${dc}:${idx}`
if (key) {
this.state.authKeysTemp.set(k, key)
this.state.authKeysTempExpiry.set(k, expires)
} else {
this.state.authKeysTemp.delete(k)
this.state.authKeysTempExpiry.delete(k)
}
}
getTemp(dc: number, idx: number, now: number): Uint8Array | null {
const k = `${dc}:${idx}`
if (now > (this.state.authKeysTempExpiry.get(k) ?? 0)) {
return null
}
return this.state.authKeysTemp.get(k) ?? null
}
deleteByDc(dc: number): void {
this.state.authKeys.delete(dc)
for (const key of this.state.authKeysTemp.keys()) {
if (key.startsWith(`${dc}:`)) {
this.state.authKeysTemp.delete(key)
this.state.authKeysTempExpiry.delete(key)
}
}
}
deleteAll(): void {
this.state.authKeys.clear()
this.state.authKeysTemp.clear()
this.state.authKeysTempExpiry.clear()
}
}

24
packages/core/src/storage/providers/memory/repository/kv.ts Normal file
View file

@ -0,0 +1,24 @@
import { IKeyValueRepository } from '../../../repository/key-value.js'
import { MemoryStorageDriver } from '../driver.js'
export class MemoryKeyValueRepository implements IKeyValueRepository {
constructor(readonly _driver: MemoryStorageDriver) {}
readonly state = this._driver.getState<Map<string, Uint8Array>>('kv', new Map())
set(key: string, value: Uint8Array): void {
this.state.set(key, value)
}
get(key: string): Uint8Array | null {
return this.state.get(key) ?? null
}
delete(key: string): void {
this.state.delete(key)
}
deleteAll(): void {
this.state.clear()
}
}

67
packages/core/src/storage/providers/memory/repository/peers.ts Normal file
View file

@ -0,0 +1,67 @@
import { IPeersRepository } from '../../../repository/peers.js'
import { MemoryStorageDriver } from '../driver.js'
interface PeersState {
entities: Map<number, IPeersRepository.PeerInfo>
usernameIndex: Map<string, number>
phoneIndex: Map<string, number>
}
export class MemoryPeersRepository implements IPeersRepository {
constructor(readonly _driver: MemoryStorageDriver) {}
readonly state = this._driver.getState<PeersState>('peers', {
entities: new Map(),
usernameIndex: new Map(),
phoneIndex: new Map(),
})
store(peer: IPeersRepository.PeerInfo): void {
const old = this.state.entities.get(peer.id)
if (old) {
// delete old index entries if needed
old.usernames.forEach((username) => {
this.state.usernameIndex.delete(username)
})
if (old.phone) {
this.state.phoneIndex.delete(old.phone)
}
}
if (peer.usernames) {
for (const username of peer.usernames) {
this.state.usernameIndex.set(username, peer.id)
}
}
if (peer.phone) this.state.phoneIndex.set(peer.phone, peer.id)
this.state.entities.set(peer.id, peer)
}
getById(id: number): IPeersRepository.PeerInfo | null {
return this.state.entities.get(id) ?? null
}
getByUsername(username: string): IPeersRepository.PeerInfo | null {
const id = this.state.usernameIndex.get(username.toLowerCase())
if (!id) return null
return this.state.entities.get(id) ?? null
}
getByPhone(phone: string): IPeersRepository.PeerInfo | null {
const id = this.state.phoneIndex.get(phone)
if (!id) return null
return this.state.entities.get(id) ?? null
}
deleteAll(): void {
this.state.entities.clear()
this.state.phoneIndex.clear()
this.state.usernameIndex.clear()
}
}

49
packages/core/src/storage/providers/memory/repository/ref-messages.ts Normal file
View file

@ -0,0 +1,49 @@
import { IReferenceMessagesRepository } from '../../../repository/ref-messages.js'
import { MemoryStorageDriver } from '../driver.js'
interface RefMessagesState {
refs: Map<number, Set<string>>
}
export class MemoryRefMessagesRepository implements IReferenceMessagesRepository {
constructor(readonly _driver: MemoryStorageDriver) {}
readonly state = this._driver.getState<RefMessagesState>('refMessages', {
refs: new Map(),
})
store(peerId: number, chatId: number, msgId: number): void {
if (!this.state.refs.has(peerId)) {
this.state.refs.set(peerId, new Set())
}
this.state.refs.get(peerId)!.add(`${chatId}:${msgId}`)
}
getByPeer(peerId: number): [number, number] | null {
const refs = this.state.refs.get(peerId)
if (!refs?.size) return null
const [ref] = refs
const [chatId, msgId] = ref.split(':')
return [Number(chatId), Number(msgId)]
}
delete(chatId: number, msgIds: number[]): void {
// not the most efficient way, but it's fine
for (const refs of this.state.refs.values()) {
for (const msg of msgIds) {
refs.delete(`${chatId}:${msg}`)
}
}
}
deleteByPeer(peerId: number): void {
this.state.refs.delete(peerId)
}
deleteAll(): void {
this.state.refs.clear()
}
}

113
packages/core/src/storage/repository/auth-keys.test-utils.ts Normal file
View file

@ -0,0 +1,113 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { IAuthKeysRepository } from './auth-keys.js'
export function fakeAuthKeysRepository(): IAuthKeysRepository {
return {
get: vi.fn(),
set: vi.fn(),
getTemp: vi.fn(),
setTemp: vi.fn(),
deleteByDc: vi.fn(),
deleteAll: vi.fn(),
}
}
function fixBuffer(buf: Uint8Array | null): Uint8Array | null {
if (!buf) return buf
// eslint-disable-next-line no-restricted-globals
return typeof Buffer !== 'undefined' && buf instanceof Buffer ? new Uint8Array(buf) : buf
}
export function testAuthKeysRepository(repo: IAuthKeysRepository) {
const key2 = new Uint8Array(256).fill(0x42)
const key3 = new Uint8Array(256).fill(0x43)
const key2i0 = new Uint8Array(256).fill(0x44)
const key2i1 = new Uint8Array(256).fill(0x45)
const key3i0 = new Uint8Array(256).fill(0x46)
const key3i1 = new Uint8Array(256).fill(0x47)
describe('auth keys', () => {
afterEach(() => repo.deleteAll())
it('should be empty by default', async () => {
expect(fixBuffer(await repo.get(2))).toEqual(null)
expect(fixBuffer(await repo.get(3))).toEqual(null)
})
it('should store and retrieve auth keys', async () => {
await repo.set(2, key2)
await repo.set(3, key3)
expect(fixBuffer(await repo.get(2))).toEqual(key2)
expect(fixBuffer(await repo.get(3))).toEqual(key3)
})
it('should delete auth keys', async () => {
await repo.set(2, key2)
await repo.set(3, key3)
await repo.set(2, null)
await repo.set(3, null)
expect(fixBuffer(await repo.get(2))).toEqual(null)
expect(fixBuffer(await repo.get(3))).toEqual(null)
})
it('should store and retrieve temp auth keys', async () => {
await repo.setTemp(2, 0, key2i0, 1)
await repo.setTemp(2, 1, key2i1, 1)
await repo.setTemp(3, 0, key3i0, 1)
await repo.setTemp(3, 1, key3i1, 1)
expect(fixBuffer(await repo.getTemp(2, 0, 0))).toEqual(key2i0)
expect(fixBuffer(await repo.getTemp(2, 1, 0))).toEqual(key2i1)
expect(fixBuffer(await repo.getTemp(3, 0, 0))).toEqual(key3i0)
expect(fixBuffer(await repo.getTemp(3, 1, 0))).toEqual(key3i1)
expect(fixBuffer(await repo.getTemp(2, 0, 100))).toEqual(null)
expect(fixBuffer(await repo.getTemp(2, 1, 100))).toEqual(null)
expect(fixBuffer(await repo.getTemp(3, 0, 100))).toEqual(null)
expect(fixBuffer(await repo.getTemp(3, 1, 100))).toEqual(null)
})
it('should delete temp auth keys', async () => {
await repo.setTemp(2, 0, key2i0, 1)
await repo.setTemp(2, 1, key2i1, 1)
await repo.setTemp(3, 0, key3i0, 1)
await repo.setTemp(3, 1, key3i1, 1)
await repo.setTemp(2, 0, null, 1)
await repo.setTemp(2, 1, null, 1)
await repo.setTemp(3, 0, null, 1)
await repo.setTemp(3, 1, null, 1)
expect(fixBuffer(await repo.getTemp(2, 0, 0))).toEqual(null)
expect(fixBuffer(await repo.getTemp(2, 1, 0))).toEqual(null)
expect(fixBuffer(await repo.getTemp(3, 0, 0))).toEqual(null)
expect(fixBuffer(await repo.getTemp(3, 1, 0))).toEqual(null)
})
it('should delete all auth keys by DC', async () => {
await repo.set(2, key2)
await repo.set(3, key3)
await repo.setTemp(2, 0, key2i0, 1)
await repo.setTemp(2, 1, key2i1, 1)
await repo.setTemp(3, 0, key3i0, 1)
await repo.setTemp(3, 1, key3i1, 1)
await repo.deleteByDc(2)
expect(fixBuffer(await repo.get(2))).toEqual(null)
expect(fixBuffer(await repo.get(3))).toEqual(key3)
expect(fixBuffer(await repo.getTemp(2, 0, 0))).toEqual(null)
expect(fixBuffer(await repo.getTemp(2, 1, 0))).toEqual(null)
expect(fixBuffer(await repo.getTemp(3, 0, 0))).toEqual(key3i0)
expect(fixBuffer(await repo.getTemp(3, 1, 0))).toEqual(key3i1)
})
})
}

45
packages/core/src/storage/repository/auth-keys.ts Normal file
View file

@ -0,0 +1,45 @@
import { MaybeAsync } from '../../types/utils.js'
export interface IAuthKeysRepository {
/**
* Store auth_key for the given DC
*
* If `key` is `null`, the key should be deleted instead
*
* **MUST** be applied immediately, without batching
*/
set(dc: number, key: Uint8Array | null): MaybeAsync<void>
/** Get auth_key for the given DC */
get(dc: number): MaybeAsync<Uint8Array | null>
/**
* Store temp_auth_key for the given DC and idx,
* along with its expiration date (in seconds)
*
* If `key` is `null`, the key should be deleted instead
*
* **MUST** be applied immediately, without batching
*/
setTemp(dc: number, idx: number, key: Uint8Array | null, expires: number): MaybeAsync<void>
/**
* Given the DC id, idx and point in time (in seconds),
* return the temp_auth_key that should be used for the next request
* (such that `now < key.expires`), or `null` if no such key exists
*/
getTemp(dc: number, idx: number, now: number): MaybeAsync<Uint8Array | null>
/**
* Delete all stored auth keys for the given DC, including
* both permanent and temp keys
*
* **MUST** be applied immediately, without batching
*/
deleteByDc(dc: number): MaybeAsync<void>
/**
* Delete all stored auth keys, including both permanent and temp keys
*
* **MUST** be applied immediately, without batching
*/
deleteAll(): MaybeAsync<void>
}

4
packages/core/src/storage/repository/index.ts Normal file
View file

@ -0,0 +1,4 @@
export * from './auth-keys.js'
export * from './key-value.js'
export * from './peers.js'
export * from './ref-messages.js'

47
packages/core/src/storage/repository/key-value.test-utils.ts Normal file
View file

@ -0,0 +1,47 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { IStorageDriver } from '../driver.js'
import { IKeyValueRepository } from './key-value.js'
export function fakeKeyValueRepository(): IKeyValueRepository {
return {
get: vi.fn(),
set: vi.fn(),
delete: vi.fn(),
deleteAll: vi.fn(),
}
}
function fixBuffer(buf: Uint8Array | null): Uint8Array | null {
if (!buf) return buf
// eslint-disable-next-line no-restricted-globals
return typeof Buffer !== 'undefined' && buf instanceof Buffer ? new Uint8Array(buf) : buf
}
export function testKeyValueRepository(repo: IKeyValueRepository, driver: IStorageDriver) {
describe('key-value', () => {
afterEach(() => repo.deleteAll())
it('should be empty by default', async () => {
expect(fixBuffer(await repo.get('key'))).toEqual(null)
})
it('should store and retrieve values', async () => {
await repo.set('key', new Uint8Array([1, 2, 3]))
await driver.save?.()
expect(fixBuffer(await repo.get('key'))).toEqual(new Uint8Array([1, 2, 3]))
})
it('should delete values', async () => {
await repo.set('key', new Uint8Array([1, 2, 3]))
await driver.save?.()
await repo.delete('key')
await driver.save?.()
expect(fixBuffer(await repo.get('key'))).toEqual(null)
})
})
}

12
packages/core/src/storage/repository/key-value.ts Normal file
View file

@ -0,0 +1,12 @@
import { MaybeAsync } from '../../types/utils.js'
export interface IKeyValueRepository {
/** Set a key-value pair */
set(key: string, value: Uint8Array): MaybeAsync<void>
/** Get a key-value pair */
get(key: string): MaybeAsync<Uint8Array | null>
/** Delete a key-value pair */
delete(key: string): MaybeAsync<void>
deleteAll(): MaybeAsync<void>
}

84
packages/core/src/storage/repository/peers.test-utils.ts Normal file
View file

@ -0,0 +1,84 @@
import { describe, expect, it, vi } from 'vitest'
import { createStub } from '@mtcute/test'
import { __tlWriterMap } from '@mtcute/tl/binary/writer.js'
import { TlBinaryWriter } from '@mtcute/tl-runtime'
import { IStorageDriver } from '../driver.js'
import { IPeersRepository } from './peers.js'
export function fakePeersRepository(): IPeersRepository {
return {
getById: vi.fn(),
getByUsername: vi.fn(),
getByPhone: vi.fn(),
store: vi.fn(),
deleteAll: vi.fn(),
}
}
function fixPeerInfo(peer: IPeersRepository.PeerInfo | null): IPeersRepository.PeerInfo | null {
if (!peer) return peer
return {
...peer,
complete:
// eslint-disable-next-line no-restricted-globals
typeof Buffer !== 'undefined' && peer.complete instanceof Buffer ?
new Uint8Array(peer.complete) :
peer.complete,
}
}
export function testPeersRepository(repo: IPeersRepository, driver: IStorageDriver) {
const stubPeerUser: IPeersRepository.PeerInfo = {
id: 123123,
accessHash: '123|456',
usernames: ['some_user'],
phone: '78005553535',
updated: 666,
complete: TlBinaryWriter.serializeObject(__tlWriterMap, createStub('user', { id: 123123 })),
}
const stubPeerChannel: IPeersRepository.PeerInfo = {
id: -1001183945448,
accessHash: '666|555',
usernames: ['some_channel'],
updated: 777,
complete: TlBinaryWriter.serializeObject(__tlWriterMap, createStub('channel', { id: 123123 })),
}
describe('peers', () => {
it('should be empty by default', async () => {
expect(await repo.getById(123123)).toEqual(null)
expect(await repo.getByUsername('some_user')).toEqual(null)
expect(await repo.getByPhone('phone')).toEqual(null)
})
it('should store and retrieve peers', async () => {
repo.store(stubPeerUser)
repo.store(stubPeerChannel)
await driver.save?.()
expect(fixPeerInfo(await repo.getById(123123))).toEqual(stubPeerUser)
expect(fixPeerInfo(await repo.getByUsername('some_user'))).toEqual(stubPeerUser)
expect(fixPeerInfo(await repo.getByPhone('78005553535'))).toEqual(stubPeerUser)
expect(fixPeerInfo(await repo.getById(-1001183945448))).toEqual(stubPeerChannel)
expect(fixPeerInfo(await repo.getByUsername('some_channel'))).toEqual(stubPeerChannel)
})
it('should update peers usernames', async () => {
repo.store(stubPeerUser)
await driver.save?.()
const modUser = { ...stubPeerUser, usernames: ['some_user2'] }
repo.store(modUser)
await driver.save?.()
expect(fixPeerInfo(await repo.getById(123123))).toEqual(modUser)
expect(await repo.getByUsername('some_user')).toEqual(null)
expect(fixPeerInfo(await repo.getByUsername('some_user2'))).toEqual(modUser)
})
})
}

37
packages/core/src/storage/repository/peers.ts Normal file
View file

@ -0,0 +1,37 @@
import { MaybeAsync } from '../../types/utils.js'
// eslint-disable-next-line @typescript-eslint/no-namespace
export namespace IPeersRepository {
/** Information about a cached peer */
export interface PeerInfo {
/** Peer marked ID */
id: number
/** Peer access hash, as a fast string representation */
accessHash: string
/** Peer usernames, if any */
usernames: string[]
/** Timestamp (in seconds) when the peer was last updated */
updated: number
/** Peer phone number, if available */
phone?: string
/**
* Complete information about the peer,
* serialization of {@link tl.TypeUser} or {@link tl.TypeChat}
*/
complete: Uint8Array
}
}
export interface IPeersRepository {
/** Store the given peer*/ // todo remove any reference messages linked to them
store(peer: IPeersRepository.PeerInfo): MaybeAsync<void>
/** Find a peer by their `id` */
getById(id: number): MaybeAsync<IPeersRepository.PeerInfo | null>
/** Find a peer by their username (where `usernames` includes `username`) */
getByUsername(username: string): MaybeAsync<IPeersRepository.PeerInfo | null>
/** Find a peer by their `phone` */
getByPhone(phone: string): MaybeAsync<IPeersRepository.PeerInfo | null>
deleteAll(): MaybeAsync<void>
}

70
packages/core/src/storage/repository/ref-messages.test-utils.ts Normal file
View file

@ -0,0 +1,70 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { IStorageDriver } from '../driver.js'
import { IReferenceMessagesRepository } from './ref-messages.js'
export function fakeRefMessagesRepository(): IReferenceMessagesRepository {
return {
store: vi.fn(),
getByPeer: vi.fn(),
delete: vi.fn(),
deleteByPeer: vi.fn(),
deleteAll: vi.fn(),
}
}
export function testRefMessagesRepository(repo: IReferenceMessagesRepository, driver: IStorageDriver) {
describe('IReferenceMessagesRepository', () => {
afterEach(() => repo.deleteAll())
it('should be empty by default', async () => {
expect(await repo.getByPeer(1)).toEqual(null)
})
it('should store and retrieve reference messages', async () => {
await repo.store(1, 2, 3)
await repo.store(1, 4, 5)
await repo.store(2, 6, 7)
await driver.save?.()
expect(await repo.getByPeer(1)).deep.oneOf([[2, 3], [4, 5]])
expect(await repo.getByPeer(2)).toEqual([6, 7])
expect(await repo.getByPeer(3)).toEqual(null)
expect(await repo.getByPeer(4)).toEqual(null)
expect(await repo.getByPeer(5)).toEqual(null)
expect(await repo.getByPeer(6)).toEqual(null)
expect(await repo.getByPeer(7)).toEqual(null)
})
it('should delete reference messages', async () => {
await repo.store(1, 2, 3)
await repo.store(1, 4, 5)
await repo.store(2, 6, 7)
await driver.save?.()
await repo.delete(4, [5])
await driver.save?.()
expect(await repo.getByPeer(1)).toEqual([2, 3])
await repo.delete(2, [2, 3, 4])
await driver.save?.()
expect(await repo.getByPeer(1)).toEqual(null)
})
it('should delete all reference messages for a peer', async () => {
await repo.store(1, 2, 3)
await repo.store(1, 4, 5)
await repo.store(1, 6, 7)
await repo.store(2, 20, 30)
await repo.store(2, 40, 50)
await repo.store(2, 60, 70)
await driver.save?.()
await repo.deleteByPeer(1)
await driver.save?.()
expect(await repo.getByPeer(1)).toEqual(null)
expect(await repo.getByPeer(2)).deep.oneOf([[20, 30], [40, 50], [60, 70]])
})
})
}

22
packages/core/src/storage/repository/ref-messages.ts Normal file
View file

@ -0,0 +1,22 @@
import { MaybeAsync } from '../../types/utils.js'
export interface IReferenceMessagesRepository {
/** Store a reference message */
store(peerId: number, chatId: number, msgId: number): MaybeAsync<void>
/**
* Get the reference message for the given `peerId`.
*
* If more than one reference message is stored for the given `peerId`,
* the one with the highest `msgId` should be returned, but this is not
* really important.
*/
getByPeer(peerId: number): MaybeAsync<[number, number] | null>
/**
* Delete reference messages given the `chatId`
* where `msgId` is one of `msgIds`
*/
delete(chatId: number, msgIds: number[]): MaybeAsync<void>
deleteByPeer(peerId: number): MaybeAsync<void>
deleteAll(): MaybeAsync<void>
}

26
packages/core/src/storage/service/auth-keys.test.ts Normal file
View file

@ -0,0 +1,26 @@
import { describe, expect, it, vi } from 'vitest'
import { fakeAuthKeysRepository } from '../repository/auth-keys.test-utils.js'
import { fakeKeyValueRepository } from '../repository/key-value.test-utils.js'
import { AuthKeysService } from './auth-keys.js'
import { FutureSaltsService } from './future-salts.js'
import { testServiceOptions } from './utils.test-utils.js'
describe('auth keys service', () => {
const fakeKeys = fakeAuthKeysRepository()
const fakeKv = fakeKeyValueRepository()
describe('deleteByDc', () => {
it('should delete keys and salts for given DC', async () => {
const saltsService = new FutureSaltsService(fakeKv, testServiceOptions())
const service = new AuthKeysService(fakeKeys, saltsService, testServiceOptions())
vi.spyOn(saltsService, 'delete')
await service.deleteByDc(2)
expect(fakeKeys.deleteByDc).toHaveBeenCalledWith(2)
expect(saltsService.delete).toHaveBeenCalledWith(2)
})
})
})

18
packages/core/src/storage/service/auth-keys.ts Normal file
View file

@ -0,0 +1,18 @@
import { IAuthKeysRepository } from '../repository/auth-keys.js'
import { BaseService, ServiceOptions } from './base.js'
import { FutureSaltsService } from './future-salts.js'
export class AuthKeysService extends BaseService {
constructor(
readonly _keys: IAuthKeysRepository,
readonly _salts: FutureSaltsService,
opts: ServiceOptions,
) {
super(opts)
}
async deleteByDc(dc: number): Promise<void> {
await this._keys.deleteByDc(dc)
await this._salts.delete(dc)
}
}

38
packages/core/src/storage/service/base.ts Normal file
View file

@ -0,0 +1,38 @@
import { tl } from '@mtcute/tl'
import { TlBinaryReader, TlBinaryWriter, TlReaderMap, TlWriterMap } from '@mtcute/tl-runtime'
import { Logger } from '../../utils/logger.js'
import { IStorageDriver } from '../driver.js'
export interface ServiceOptions {
driver: IStorageDriver
readerMap: TlReaderMap
writerMap: TlWriterMap
log: Logger
}
export class BaseService {
readonly _driver: IStorageDriver
readonly _readerMap: TlReaderMap
readonly _writerMap: TlWriterMap
readonly _log: Logger
constructor(opts: ServiceOptions) {
this._driver = opts.driver
this._readerMap = opts.readerMap
this._writerMap = opts.writerMap
this._log = opts.log
}
protected _serializeTl(obj: tl.TlObject): Uint8Array {
return TlBinaryWriter.serializeObject(this._writerMap, obj)
}
protected _deserializeTl(data: Uint8Array): tl.TlObject | null {
try {
return TlBinaryReader.deserializeObject<tl.TlObject>(this._readerMap, data)
} catch (e) {
return null
}
}
}

96
packages/core/src/storage/service/current-user.ts Normal file
View file

@ -0,0 +1,96 @@
import { TlBinaryReader, TlBinaryWriter } from '@mtcute/tl-runtime'
import { MtArgumentError } from '../../types/index.js'
import { IKeyValueRepository } from '../repository/key-value.js'
import { BaseService, ServiceOptions } from './base.js'
export interface CurrentUserInfo {
userId: number
isBot: boolean
}
// todo: do we need this in core?
const KV_CURRENT_USER = 'current_user'
function serialize(info: CurrentUserInfo | null): Uint8Array {
if (!info) return new Uint8Array(0)
const writer = TlBinaryWriter.manual(16)
writer.int(1) // version
let flags = 0
if (info.isBot) flags |= 1
writer.int(flags)
writer.int53(info.userId)
return writer.result()
}
function parse(data: Uint8Array): CurrentUserInfo | null {
if (data.length === 0) return null
const reader = TlBinaryReader.manual(data)
if (reader.int() !== 1) return null
const flags = reader.int()
const userId = reader.int53()
return {
userId,
isBot: (flags & 1) !== 0,
}
}
// todo: add testMode here
export class CurrentUserService extends BaseService {
constructor(
readonly _kv: IKeyValueRepository,
opts: ServiceOptions,
) {
super(opts)
}
private _cached?: CurrentUserInfo | null
async store(info: CurrentUserInfo | null): Promise<void> {
if (info && this._cached) {
// update the existing object so the references to it are still valid
if (this._cached.userId === info.userId) {
return
}
this._cached.userId = info.userId
this._cached.isBot = info.isBot
} else {
this._cached = info
}
await this._kv.set(KV_CURRENT_USER, serialize(info))
await this._driver.save?.()
}
async fetch(): Promise<CurrentUserInfo | null> {
if (this._cached) return this._cached
const data = await this._kv.get(KV_CURRENT_USER)
if (!data) return null
const info = parse(data)
this._cached = info
return info
}
getCached(safe = false): CurrentUserInfo | null {
if (this._cached === undefined) {
if (safe) return null
throw new MtArgumentError('User info is not cached yet')
}
return this._cached
}
}

67
packages/core/src/storage/service/default-dcs.ts Normal file
View file

@ -0,0 +1,67 @@
import { DcOptions, parseBasicDcOption, serializeBasicDcOption } from '../../utils/dcs.js'
import { IKeyValueRepository } from '../repository/key-value.js'
import { BaseService, ServiceOptions } from './base.js'
const KV_MAIN = 'dc_main'
const KV_MEDIA = 'dc_media'
export class DefaultDcsService extends BaseService {
constructor(
readonly _kv: IKeyValueRepository,
opts: ServiceOptions,
) {
super(opts)
}
private _cached?: DcOptions
async store(dcs: DcOptions): Promise<void> {
if (this._cached) {
if (
this._cached.main === dcs.main &&
this._cached.media === dcs.media
) return
}
this._cached = dcs
const { main, media } = dcs
const mainData = serializeBasicDcOption(main)
await this._kv.set(KV_MAIN, mainData)
if (media !== main) {
const mediaData = serializeBasicDcOption(media)
await this._kv.set(KV_MEDIA, mediaData)
} else {
await this._kv.delete(KV_MEDIA)
}
}
async fetch(): Promise<DcOptions | null> {
if (this._cached) return this._cached
const [mainData, mediaData] = await Promise.all([
this._kv.get(KV_MAIN),
this._kv.get(KV_MEDIA),
])
if (!mainData) return null
const main = parseBasicDcOption(mainData)
if (!main) return null
const dcs: DcOptions = { main, media: main }
if (mediaData) {
const media = parseBasicDcOption(mediaData)
if (media) {
dcs.media = media
}
}
this._cached = dcs
return dcs
}
}

52
packages/core/src/storage/service/future-salts.ts Normal file
View file

@ -0,0 +1,52 @@
import { mtp } from '@mtcute/tl'
import { TlBinaryReader, TlBinaryWriter } from '@mtcute/tl-runtime'
import { IKeyValueRepository } from '../repository/key-value.js'
import { BaseService, ServiceOptions } from './base.js'
const KV_PREFIX = 'salts:'
export class FutureSaltsService extends BaseService {
constructor(
readonly _kv: IKeyValueRepository,
opts: ServiceOptions,
) {
super(opts)
}
private _cached = new Map<number, mtp.RawMt_future_salt[]>()
async store(dcId: number, salts: mtp.RawMt_future_salt[]): Promise<void> {
if (this._cached.get(dcId) === salts) return
const writer = TlBinaryWriter.alloc(this._writerMap, 8 + 20 * salts.length)
writer.vector(writer.object, salts)
await this._kv.set(KV_PREFIX + dcId, writer.result())
}
async fetch(dcId: number): Promise<mtp.RawMt_future_salt[] | null> {
const cached = this._cached.get(dcId)
if (cached) return cached
const data = await this._kv.get(KV_PREFIX + dcId)
if (!data) return null
const reader = new TlBinaryReader(this._readerMap, data)
const salts = reader.vector()
for (const salt of salts) {
if ((salt as { _: string })._ !== 'mt_future_salt') return null
}
const salts_ = salts as mtp.RawMt_future_salt[]
this._cached.set(dcId, salts_)
return salts_
}
async delete(dcId: number): Promise<void> {
this._cached.delete(dcId)
await this._kv.delete(KV_PREFIX + dcId)
}
}

273
packages/core/src/storage/service/peers.ts Normal file
View file

@ -0,0 +1,273 @@
import Long from 'long'
import { tl } from '@mtcute/tl'
import { longFromFastString, longToFastString } from '../../utils/long-utils.js'
import { LruMap } from '../../utils/lru-map.js'
import { getAllPeersFrom, parseMarkedPeerId, toggleChannelIdMark } from '../../utils/peer-utils.js'
import { IPeersRepository } from '../repository/peers.js'
import { BaseService, ServiceOptions } from './base.js'
import { RefMessagesService } from './ref-messages.js'
interface CacheItem {
peer: tl.TypeInputPeer
complete: tl.TypeUser | tl.TypeChat | null
}
export interface PeersServiceOptions {
cacheSize?: number
updatesWriteInterval?: number
}
// todo: move into @mtcute/client somehow?
const USERNAME_TTL = 24 * 60 * 60 * 1000 // 1 day
function getInputPeer(dto: IPeersRepository.PeerInfo): tl.TypeInputPeer {
const [type, id] = parseMarkedPeerId(dto.id)
switch (type) {
case 'user':
return {
_: 'inputPeerUser',
userId: id,
accessHash: longFromFastString(dto.accessHash),
}
case 'chat':
return {
_: 'inputPeerChat',
chatId: id,
}
case 'channel':
return {
_: 'inputPeerChannel',
channelId: id,
accessHash: longFromFastString(dto.accessHash),
}
}
}
function getUsernames(obj: tl.RawUser | tl.RawChannel) {
if (obj.usernames?.length) return obj.usernames.map((x) => x.username.toLowerCase())
if (obj.username) return [obj.username.toLowerCase()]
return []
}
export class PeersService extends BaseService {
private _cache: LruMap<number, CacheItem>
private _pendingWrites = new Map<number, IPeersRepository.PeerInfo>()
constructor(
readonly options: PeersServiceOptions,
readonly _peers: IPeersRepository,
readonly _refs: RefMessagesService,
common: ServiceOptions,
) {
super(common)
this._cache = new LruMap(options.cacheSize ?? 100)
}
async updatePeersFrom(obj: tl.TlObject | tl.TlObject[]) {
let count = 0
for (const peer of getAllPeersFrom(obj)) {
// no point in caching min peers as we can't use them
if ((peer as Extract<typeof peer, { min?: unknown }>).min) continue
count += 1
await this.store(peer)
}
if (count > 0) {
await this._driver.save?.()
this._log.debug('cached %d peers', count)
return true
}
return false
}
async store(peer: tl.TypeUser | tl.TypeChat): Promise<void> {
let dto: IPeersRepository.PeerInfo
let accessHash: tl.Long
switch (peer._) {
case 'user': {
if (!peer.accessHash) {
this._log.warn('received user without access hash: %j', peer)
return
}
dto = {
id: peer.id,
accessHash: longToFastString(peer.accessHash),
phone: peer.phone,
usernames: getUsernames(peer),
updated: Date.now(),
complete: this._serializeTl(peer),
}
accessHash = peer.accessHash
break
}
case 'chat':
case 'chatForbidden': {
dto = {
id: -peer.id,
accessHash: '',
updated: Date.now(),
complete: this._serializeTl(peer),
usernames: [],
}
accessHash = Long.ZERO
break
}
case 'channel':
case 'channelForbidden': {
if (!peer.accessHash) {
this._log.warn('received channel without access hash: %j', peer)
return
}
dto = {
id: toggleChannelIdMark(peer.id),
accessHash: longToFastString(peer.accessHash),
usernames: getUsernames(peer as tl.RawChannel),
updated: Date.now(),
complete: this._serializeTl(peer),
}
accessHash = peer.accessHash
break
}
default:
return
}
const cached = this._cache.get(peer.id)
if (cached && this.options.updatesWriteInterval !== 0) {
const oldAccessHash = (cached.peer as Extract<tl.TypeInputPeer, { accessHash?: unknown }>).accessHash
if (oldAccessHash?.eq(accessHash)) {
// when entity is cached and hash is the same, an update query is needed,
// since some field in the full entity might have changed, or the username/phone
//
// to avoid too many DB calls, and since these updates are pretty common,
// they are grouped and applied in batches no more than once every 30sec (or user-defined).
//
// until then, they are either served from in-memory cache,
// or an older version is fetched from DB
this._pendingWrites.set(peer.id, dto)
cached.complete = peer
return
}
}
// entity is not cached in memory, or the access hash has changed
// we need to update it in the DB asap, and also update the in-memory cache
await this._peers.store(dto)
this._cache.set(peer.id, {
peer: getInputPeer(dto),
complete: peer,
})
// todo: if (!this._cachedSelf?.isBot) {
// we have the full peer, we no longer need the references
// we can skip this in the other branch, since in that case it would've already been deleted
await this._refs.deleteByPeer(peer.id)
}
private _returnCaching(id: number, dto: IPeersRepository.PeerInfo) {
const peer = getInputPeer(dto)
const complete = this._deserializeTl(dto.complete)
this._cache.set(id, {
peer,
complete: complete as tl.TypeUser | tl.TypeChat | null,
})
return peer
}
async getById(id: number, allowRefs = true): Promise<tl.TypeInputPeer | null> {
const cached = this._cache.get(id)
if (cached) return cached.peer
const dto = await this._peers.getById(id)
if (dto) {
return this._returnCaching(id, dto)
}
if (allowRefs) {
const ref = await this._refs.getForPeer(id)
if (!ref) return null
const [chatId, msgId] = ref
const chat = await this.getById(chatId, false)
if (!chat) return null
if (id > 0) {
// user
return {
_: 'inputPeerUserFromMessage',
peer: chat,
msgId,
userId: id,
}
}
// channel
return {
_: 'inputPeerChannelFromMessage',
peer: chat,
msgId,
channelId: toggleChannelIdMark(id),
}
}
return null
}
async getByPhone(phone: string): Promise<tl.TypeInputPeer | null> {
const dto = await this._peers.getByPhone(phone)
if (!dto) return null
return this._returnCaching(dto.id, dto)
}
async getByUsername(username: string): Promise<tl.TypeInputPeer | null> {
const dto = await this._peers.getByUsername(username.toLowerCase())
if (!dto) return null
if (Date.now() - dto.updated > USERNAME_TTL) {
// username is too old, we can't trust it. ask the client to re-fetch it
return null
}
return this._returnCaching(dto.id, dto)
}
async getCompleteById(id: number): Promise<tl.TypeUser | tl.TypeChat | null> {
const cached = this._cache.get(id)
if (cached) return cached.complete
const dto = await this._peers.getById(id)
if (!dto) return null
const cacheItem: CacheItem = {
peer: getInputPeer(dto),
complete: this._deserializeTl(dto.complete) as tl.TypeUser | tl.TypeChat | null,
}
this._cache.set(id, cacheItem)
return cacheItem.complete
}
}

51
packages/core/src/storage/service/ref-messages.ts Normal file
View file

@ -0,0 +1,51 @@
import { LruMap } from '../../utils/lru-map.js'
import { IReferenceMessagesRepository } from '../repository/ref-messages.js'
import { BaseService, ServiceOptions } from './base.js'
export interface RefMessagesServiceOptions {
cacheSize?: number
}
// todo: move inside updates manager?
// todo: chatId -> channelId
export class RefMessagesService extends BaseService {
private _cache: LruMap<number, [number, number]>
constructor(
readonly options: RefMessagesServiceOptions,
readonly _refs: IReferenceMessagesRepository,
common: ServiceOptions,
) {
super(common)
this._cache = new LruMap(options.cacheSize ?? 1000)
}
async store(peerId: number, chatId: number, msgId: number): Promise<void> {
await this._refs.store(peerId, chatId, msgId)
this._cache.set(peerId, [chatId, msgId])
}
async getForPeer(peerId: number): Promise<[number, number] | null> {
const cached = this._cache.get(peerId)
if (cached) return cached
const ref = await this._refs.getByPeer(peerId)
if (ref) this._cache.set(peerId, ref)
return ref
}
async delete(chatId: number, msgIds: number[]): Promise<void> {
await this._refs.delete(chatId, msgIds)
// it's too expensive to invalidate the cache,
// so we just clear it completely instead
this._cache.clear()
}
async deleteByPeer(peerId: number): Promise<void> {
await this._refs.deleteByPeer(peerId)
this._cache.delete(peerId)
}
}

82
packages/core/src/storage/service/updates.test.ts Normal file
View file

@ -0,0 +1,82 @@
import { describe, expect, it, vi } from 'vitest'
import { fakeKeyValueRepository } from '../repository/key-value.test-utils.js'
import { UpdatesStateService } from './updates.js'
import { testServiceOptions } from './utils.test-utils.js'
describe('updates state service', () => {
const kv = fakeKeyValueRepository()
const service = new UpdatesStateService(kv, testServiceOptions())
it('should write pts to updates_pts key', async () => {
await service.setPts(123)
expect(kv.set).toHaveBeenCalledWith('updates_pts', new Uint8Array([123, 0, 0, 0]))
})
it('should write qts to updates_qts key', async () => {
await service.setQts(123)
expect(kv.set).toHaveBeenCalledWith('updates_qts', new Uint8Array([123, 0, 0, 0]))
})
it('should write date to updates_date key', async () => {
await service.setDate(123)
expect(kv.set).toHaveBeenCalledWith('updates_date', new Uint8Array([123, 0, 0, 0]))
})
it('should write seq to updates_seq key', async () => {
await service.setSeq(123)
expect(kv.set).toHaveBeenCalledWith('updates_seq', new Uint8Array([123, 0, 0, 0]))
})
describe('getState', () => {
it('should read from updates_* keys', async () => {
await service.getState()
expect(kv.get).toHaveBeenCalledWith('updates_pts')
expect(kv.get).toHaveBeenCalledWith('updates_qts')
expect(kv.get).toHaveBeenCalledWith('updates_date')
expect(kv.get).toHaveBeenCalledWith('updates_seq')
})
it('should return null if no state is stored', async () => {
vi.mocked(kv.get).mockResolvedValueOnce(null)
expect(await service.getState()).toEqual(null)
})
})
describe('getChannelPts', () => {
it('should read from updates_channel:xxx key', async () => {
await service.getChannelPts(123)
expect(kv.get).toHaveBeenCalledWith('updates_channel:123')
})
it('should return null if no value is stored', async () => {
vi.mocked(kv.get).mockResolvedValueOnce(null)
expect(await service.getChannelPts(123)).toEqual(null)
})
it('should return the value if it is stored', async () => {
vi.mocked(kv.get).mockResolvedValueOnce(new Uint8Array([1, 2, 3, 4]))
expect(await service.getChannelPts(123)).toEqual(0x04030201)
})
})
describe('setChannelPts', () => {
it('should write to updates_channel:xxx key', async () => {
await service.setChannelPts(123, 0x04030201)
expect(kv.set).toHaveBeenCalledWith(
'updates_channel:123',
new Uint8Array([1, 2, 3, 4]),
)
})
})
})

89
packages/core/src/storage/service/updates.ts Normal file
View file

@ -0,0 +1,89 @@
import { dataViewFromBuffer } from '../../utils/buffer-utils.js'
import { IKeyValueRepository } from '../repository/key-value.js'
import { BaseService, ServiceOptions } from './base.js'
const KV_PTS = 'updates_pts'
const KV_QTS = 'updates_qts'
const KV_DATE = 'updates_date'
const KV_SEQ = 'updates_seq'
const KV_CHANNEL_PREFIX = 'updates_channel:'
// todo: move inside updates manager?
export class UpdatesStateService extends BaseService {
constructor(
readonly _kv: IKeyValueRepository,
opts: ServiceOptions,
) {
super(opts)
}
private async _getInt(key: string): Promise<number | null> {
const val = await this._kv.get(key)
if (!val) return null
return dataViewFromBuffer(val).getInt32(0, true)
}
private async _setInt(key: string, val: number): Promise<void> {
const buf = new Uint8Array(4)
dataViewFromBuffer(buf).setInt32(0, val, true)
await this._kv.set(key, buf)
}
async getState(): Promise<[number, number, number, number] | null> {
const [pts, qts, date, seq] = await Promise.all([
this._getInt(KV_PTS),
this._getInt(KV_QTS),
this._getInt(KV_DATE),
this._getInt(KV_SEQ),
])
if (pts === null || qts === null || date === null || seq === null) {
return null
}
return [pts, qts, date, seq]
}
async setPts(pts: number): Promise<void> {
await this._setInt(KV_PTS, pts)
}
async setQts(qts: number): Promise<void> {
await this._setInt(KV_QTS, qts)
}
async setDate(date: number): Promise<void> {
await this._setInt(KV_DATE, date)
}
async setSeq(seq: number): Promise<void> {
await this._setInt(KV_SEQ, seq)
}
async getChannelPts(channelId: number): Promise<number | null> {
const val = await this._kv.get(KV_CHANNEL_PREFIX + channelId)
if (!val) return null
return dataViewFromBuffer(val).getUint32(0, true)
}
async setChannelPts(channelId: number, pts: number): Promise<void> {
const buf = new Uint8Array(4)
dataViewFromBuffer(buf).setUint32(0, pts, true)
await this._kv.set(KV_CHANNEL_PREFIX + channelId, buf)
}
async setManyChannelPts(cpts: Map<number, number>): Promise<void> {
const promises: Promise<void>[] = []
for (const [channelId, pts] of cpts.entries()) {
promises.push(this.setChannelPts(channelId, pts))
}
await Promise.all(promises)
}
}

18
packages/core/src/storage/service/utils.test-utils.ts Normal file
View file

@ -0,0 +1,18 @@
import { __tlReaderMap } from '@mtcute/tl/binary/reader.js'
import { __tlWriterMap } from '@mtcute/tl/binary/writer.js'
import { LogManager } from '../../utils/logger.js'
import { MemoryStorageDriver } from '../providers/memory/driver.js'
import { ServiceOptions } from './base.js'
export function testServiceOptions(): ServiceOptions {
const logger = new LogManager()
logger.level = 0
return {
driver: new MemoryStorageDriver(),
readerMap: __tlReaderMap,
writerMap: __tlWriterMap,
log: logger,
}
}

125
packages/core/src/storage/storage.ts Normal file
View file

@ -0,0 +1,125 @@
import { TlReaderMap, TlWriterMap } from '@mtcute/tl-runtime'
import { beforeExit } from '../utils/index.js'
import { Logger } from '../utils/logger.js'
import { IMtStorageProvider } from './provider.js'
import { AuthKeysService } from './service/auth-keys.js'
import { ServiceOptions } from './service/base.js'
import { CurrentUserService } from './service/current-user.js'
import { DefaultDcsService } from './service/default-dcs.js'
import { FutureSaltsService } from './service/future-salts.js'
import { PeersService, PeersServiceOptions } from './service/peers.js'
import { RefMessagesService, RefMessagesServiceOptions } from './service/ref-messages.js'
import { UpdatesStateService } from './service/updates.js'
/**
* Options for {@link StorageManager}, for internal use only.
*/
export interface StorageManagerOptions {
provider: IMtStorageProvider
log: Logger
readerMap: TlReaderMap
writerMap: TlWriterMap
}
/**
* Additional options for {@link StorageManager}, that
* can be customized by the user.
*/
export interface StorageManagerExtraOptions {
/**
* Interval in milliseconds for saving the storage.
*
* When saving, the storage is expected to persist
* all changes to disk, so that they are not lost.
*/
saveInterval?: number
refMessages?: RefMessagesServiceOptions
peers?: PeersServiceOptions
/**
* Whether to finalize database before exiting.
*
* @default `true`
*/
cleanup?: boolean
}
export class StorageManager {
constructor(readonly options: StorageManagerOptions & StorageManagerExtraOptions) {}
readonly provider = this.options.provider
readonly driver = this.provider.driver
readonly log = this.options.log.create('storage')
private _serviceOptions: ServiceOptions = {
driver: this.driver,
readerMap: this.options.readerMap,
writerMap: this.options.writerMap,
log: this.log,
}
readonly dcs = new DefaultDcsService(this.provider.kv, this._serviceOptions)
readonly salts = new FutureSaltsService(this.provider.kv, this._serviceOptions)
readonly updates = new UpdatesStateService(this.provider.kv, this._serviceOptions)
readonly self = new CurrentUserService(this.provider.kv, this._serviceOptions)
readonly keys = new AuthKeysService(this.provider.authKeys, this.salts, this._serviceOptions)
readonly refMsgs = new RefMessagesService(
this.options.refMessages ?? {},
this.provider.refMessages,
this._serviceOptions,
)
readonly peers = new PeersService(this.options.peers ?? {}, this.provider.peers, this.refMsgs, this._serviceOptions)
private _cleanupRestore?: () => void
private _loadPromise?: Promise<void> | true
load(): Promise<void> {
if (this._loadPromise === true) return Promise.resolve()
if (this._loadPromise) return this._loadPromise
this.driver.setup?.(this.log)
if (this.options.cleanup ?? true) {
this._cleanupRestore = beforeExit(() => this._destroy().catch((err) => this.log.error(err)))
}
this._loadPromise = Promise.resolve(this.driver.load?.()).then(() => {
this._loadPromise = true
})
return this._loadPromise
}
async save(): Promise<void> {
await this.driver.save?.()
}
async clear(withAuthKeys = false) {
if (withAuthKeys) {
await this.provider.authKeys.deleteAll()
}
await this.provider.kv.deleteAll()
await this.provider.peers.deleteAll()
await this.provider.refMessages.deleteAll()
await this.save()
}
private async _destroy(): Promise<void> {
if (!this._loadPromise) return
await this._loadPromise
await this.driver.destroy?.()
this._loadPromise = undefined
}
async destroy(): Promise<void> {
if (this._cleanupRestore) {
this._cleanupRestore()
this._cleanupRestore = undefined
}
await this._destroy()
}
}

114
packages/core/src/utils/dcs.ts Normal file
View file

@ -0,0 +1,114 @@
import { TlBinaryReader, TlBinaryWriter } from '@mtcute/tl-runtime'
export interface BasicDcOption {
ipAddress: string
port: number
id: number
ipv6?: boolean
mediaOnly?: boolean
}
export function serializeBasicDcOption(dc: BasicDcOption): Uint8Array {
const writer = TlBinaryWriter.manual(64)
const flags = (dc.ipv6 ? 1 : 0) | (dc.mediaOnly ? 2 : 0)
writer.raw(
new Uint8Array([
1, // version
dc.id,
flags,
]),
)
writer.string(dc.ipAddress)
writer.int(dc.port)
return writer.result()
}
export function parseBasicDcOption(data: Uint8Array): BasicDcOption | null {
const reader = TlBinaryReader.manual(data)
const [version, id, flags] = reader.raw(3)
if (version !== 1) return null
const ipAddress = reader.string()
const port = reader.int()
return {
id,
ipAddress,
port,
ipv6: (flags & 1) !== 0,
mediaOnly: (flags & 2) !== 0,
}
}
export interface DcOptions {
main: BasicDcOption
media: BasicDcOption
}
/** @internal */
export const defaultProductionDc: DcOptions = {
main: {
ipAddress: '149.154.167.50',
port: 443,
id: 2,
},
media: {
ipAddress: '149.154.167.222',
port: 443,
id: 2,
mediaOnly: true,
},
}
/** @internal */
export const defaultProductionIpv6Dc: DcOptions = {
main: {
ipAddress: '2001:067c:04e8:f002:0000:0000:0000:000a',
ipv6: true,
port: 443,
id: 2,
},
media: {
ipAddress: '2001:067c:04e8:f002:0000:0000:0000:000b',
ipv6: true,
port: 443,
id: 2,
mediaOnly: true,
},
}
/** @internal */
export const defaultTestDc: DcOptions = {
main: {
ipAddress: '149.154.167.40',
port: 443,
id: 2,
},
media: {
ipAddress: '149.154.167.40',
port: 443,
id: 2,
mediaOnly: true,
},
}
/** @internal */
export const defaultTestIpv6Dc: DcOptions = {
main: {
ipAddress: '2001:67c:4e8:f002::e',
port: 443,
ipv6: true,
id: 2,
},
media: {
ipAddress: '2001:67c:4e8:f002::e',
port: 443,
ipv6: true,
id: 2,
mediaOnly: true,
},
}

78
packages/core/src/utils/default-dcs.ts
View file

@ -1,78 +0,0 @@
import { ITelegramStorage } from '../storage/abstract.js'
/** @internal */
export const defaultProductionDc: ITelegramStorage.DcOptions = {
main: {
_: 'dcOption',
ipAddress: '149.154.167.50',
port: 443,
id: 2,
},
media: {
_: 'dcOption',
ipAddress: '149.154.167.222',
port: 443,
id: 2,
mediaOnly: true,
},
}
/** @internal */
export const defaultProductionIpv6Dc: ITelegramStorage.DcOptions = {
main: {
_: 'dcOption',
ipAddress: '2001:067c:04e8:f002:0000:0000:0000:000a',
ipv6: true,
port: 443,
id: 2,
},
media: {
_: 'dcOption',
ipAddress: '2001:067c:04e8:f002:0000:0000:0000:000b',
ipv6: true,
mediaOnly: true,
port: 443,
id: 2,
},
}
/** @internal */
export const defaultTestDc: ITelegramStorage.DcOptions = {
main: {
_: 'dcOption',
ipAddress: '149.154.167.40',
port: 443,
id: 2,
},
media: {
_: 'dcOption',
ipAddress: '149.154.167.40',
port: 443,
id: 2,
},
}
/** @internal */
export const defaultTestIpv6Dc: ITelegramStorage.DcOptions = {
main: {
_: 'dcOption',
ipAddress: '2001:67c:4e8:f002::e',
port: 443,
ipv6: true,
id: 2,
},
media: {
_: 'dcOption',
ipAddress: '2001:67c:4e8:f002::e',
port: 443,
ipv6: true,
id: 2,
},
}
export const defaultDcs = {
defaultTestDc,
defaultTestIpv6Dc,
defaultProductionDc,
defaultProductionIpv6Dc,
} as const

2
packages/core/src/utils/index.ts
View file

@ -4,7 +4,7 @@ export * from './buffer-utils.js'
export * from './condition-variable.js'
export * from './controllable-promise.js'
export * from './crypto/index.js'
export * from './default-dcs.js'
export * from './dcs.js'
export * from './deque.js'
export * from './early-timer.js'
export * from './function-utils.js'

31
packages/core/src/utils/peer-utils.test.ts
View file

@ -6,9 +6,8 @@ import { createStub } from '@mtcute/test'
import {
getAllPeersFrom,
getBarePeerId,
getBasicPeerType,
getMarkedPeerId,
markedPeerIdToBare,
parseMarkedPeerId,
toggleChannelIdMark,
} from './peer-utils.js'
@ -68,32 +67,18 @@ describe('getMarkedPeerId', () => {
})
})
describe('getBasicPeerType', () => {
it('should return basic peer type from Peer', () => {
expect(getBasicPeerType({ _: 'peerUser', userId: 123 })).toEqual('user')
expect(getBasicPeerType({ _: 'peerChat', chatId: 456 })).toEqual('chat')
expect(getBasicPeerType({ _: 'peerChannel', channelId: SOME_CHANNEL_ID })).toEqual('channel')
})
it('should return basic peer type from marked id', () => {
expect(getBasicPeerType(123)).toEqual('user')
expect(getBasicPeerType(-456)).toEqual('chat')
expect(getBasicPeerType(SOME_CHANNEL_ID_MARKED)).toEqual('channel')
describe('parseMarkedPeerId', () => {
it('should correctly parse marked ids', () => {
expect(parseMarkedPeerId(123)).toEqual(['user', 123])
expect(parseMarkedPeerId(-456)).toEqual(['chat', 456])
expect(parseMarkedPeerId(SOME_CHANNEL_ID_MARKED)).toEqual(['channel', SOME_CHANNEL_ID])
})
it('should throw for invalid marked ids', () => {
expect(() => getBasicPeerType(0)).toThrow('Invalid marked peer id')
expect(() => parseMarkedPeerId(0)).toThrow('Invalid marked peer id')
// secret chats are not supported yet
expect(() => getBasicPeerType(-1997852516400)).toThrow('Secret chats are not supported')
})
})
describe('markedPeerIdToBare', () => {
it('should return bare peer id from marked id', () => {
expect(markedPeerIdToBare(123)).toEqual(123)
expect(markedPeerIdToBare(-456)).toEqual(456)
expect(markedPeerIdToBare(SOME_CHANNEL_ID_MARKED)).toEqual(SOME_CHANNEL_ID)
expect(() => parseMarkedPeerId(-1997852516400)).toThrow('Secret chats are not supported')
})
})

51
packages/core/src/utils/peer-utils.ts
View file

@ -93,56 +93,27 @@ export function getMarkedPeerId(
}
/**
* Extract basic peer type from {@link tl.TypePeer} or its marked ID.
* Parse a marked ID into a {@link BasicPeerType} and a bare ID
*/
export function getBasicPeerType(peer: tl.TypePeer | number): BasicPeerType {
if (typeof peer !== 'number') {
switch (peer._) {
case 'peerUser':
return 'user'
case 'peerChat':
return 'chat'
case 'peerChannel':
return 'channel'
}
export function parseMarkedPeerId(id: number): [BasicPeerType, number] {
if (id < 0) {
if (MIN_MARKED_CHAT_ID <= id) {
return ['chat', -id]
}
if (peer < 0) {
if (MIN_MARKED_CHAT_ID <= peer) {
return 'chat'
if (MIN_MARKED_CHANNEL_ID <= id && id !== ZERO_CHANNEL_ID) {
return ['channel', ZERO_CHANNEL_ID - id]
}
if (MIN_MARKED_CHANNEL_ID <= peer && peer !== ZERO_CHANNEL_ID) {
return 'channel'
}
if (MAX_SECRET_CHAT_ID >= peer && peer !== ZERO_SECRET_CHAT_ID) {
if (MAX_SECRET_CHAT_ID >= id && id !== ZERO_SECRET_CHAT_ID) {
// return 'secret'
throw new MtUnsupportedError('Secret chats are not supported')
}
} else if (peer > 0 && peer <= MAX_USER_ID) {
return 'user'
} else if (id > 0 && id <= MAX_USER_ID) {
return ['user', id]
}
throw new MtArgumentError(`Invalid marked peer id: ${peer}`)
}
/**
* Extract bare peer ID from marked ID.
*
* @param peerId Marked peer ID
*/
export function markedPeerIdToBare(peerId: number): number {
const type = getBasicPeerType(peerId)
switch (type) {
case 'user':
return peerId
case 'chat':
return -peerId
case 'channel':
return toggleChannelIdMark(peerId)
}
throw new MtArgumentError(`Invalid marked peer id: ${id}`)
}
/**

2
packages/core/src/utils/string-session.test.ts
View file

@ -4,7 +4,7 @@ import { createStub } from '@mtcute/test'
import { __tlReaderMap } from '@mtcute/tl/binary/reader.js'
import { __tlWriterMap } from '@mtcute/tl/binary/writer.js'
import { defaultProductionDc } from './default-dcs.js'
import { defaultProductionDc } from './dcs.js'
import { readStringSession, writeStringSession } from './string-session.js'
const stubAuthKey = new Uint8Array(32)

9
packages/core/src/utils/string-session.ts
View file

@ -8,14 +8,15 @@ import {
TlWriterMap,
} from '@mtcute/tl-runtime'
import { ITelegramStorage } from '../storage/index.js'
import { CurrentUserInfo } from '../storage/service/current-user.js'
import { MtArgumentError } from '../types/index.js'
import { DcOptions } from './dcs.js'
export interface StringSessionData {
version: number
testMode: boolean
primaryDcs: ITelegramStorage.DcOptions
self?: ITelegramStorage.SelfInfo | null
primaryDcs: DcOptions
self?: CurrentUserInfo | null
authKey: Uint8Array
}
@ -85,7 +86,7 @@ export function readStringSession(readerMap: TlReaderMap, data: string): StringS
throw new MtArgumentError(`Invalid session string (dc._ = ${primaryDc._})`)
}
let self: ITelegramStorage.SelfInfo | null = null
let self: CurrentUserInfo | null = null
if (hasSelf) {
const selfId = reader.int53()

13
packages/dispatcher/src/dispatcher.ts
View file

@ -149,13 +149,14 @@ export class Dispatcher<State extends object = never> {
if (!storage) {
const _storage = client.storage
if (!isCompatibleStorage(_storage)) {
throw new MtArgumentError(
'Storage used by the client is not compatible with the dispatcher. Please provide a compatible storage manually',
)
}
// if (!isCompatibleStorage(_storage)) {
// // todo: dont throw if state is never used
// throw new MtArgumentError(
// 'Storage used by the client is not compatible with the dispatcher. Please provide a compatible storage manually',
// )
// }
storage = _storage
storage = _storage as any
}
if (storage) {

20
packages/dispatcher/src/filters/bots.test.ts
View file

@ -20,11 +20,12 @@ describe('filters.command', () => {
const ctx = createMessageContext({
message: text,
})
ctx.client.getAuthState = () => ({
isBot: true,
userId: 0,
selfUsername: 'testbot',
})
// todo
// ctx.client.getAuthState = () => ({
// isBot: true,
// userId: 0,
// selfUsername: 'testbot',
// })
// eslint-disable-next-line
if (command(...params)(ctx)) return (ctx as any).command
@ -38,10 +39,11 @@ describe('filters.command', () => {
expect(getParsedCommand('/start', ['start', 'stop'])).toEqual(['start'])
})
it('should only parse commands to the current bot', () => {
expect(getParsedCommand('/start@testbot', 'start')).toEqual(['start'])
expect(getParsedCommand('/start@otherbot', 'start')).toEqual(null)
})
// todo
// it('should only parse commands to the current bot', () => {
// expect(getParsedCommand('/start@testbot', 'start')).toEqual(['start'])
// expect(getParsedCommand('/start@otherbot', 'start')).toEqual(null)
// })
it('should parse command arguments', () => {
expect(getParsedCommand('/start foo bar baz', 'start')).toEqual(['start', 'foo', 'bar', 'baz'])

9
packages/dispatcher/src/filters/bots.ts
View file

@ -66,11 +66,12 @@ export const command = (
const lastGroup = m[m.length - 1]
if (lastGroup) {
const state = msg.client.getAuthState()
// const state = msg.client.getAuthState()
if (state.isBot && lastGroup !== state.selfUsername) {
return false
}
// if (state.isBot && lastGroup !== state.selfUsername) {
// return false
// }
console.log('todo')
}
const match = m.slice(1, -1)

3
packages/dispatcher/src/filters/chat.ts
View file

@ -89,7 +89,8 @@ export const chatId: {
case 'user_typing': {
const id = upd.chatId
return (matchSelf && id === upd.client.getAuthState().userId) || indexId.has(id)
throw new Error('TODO')
// return (matchSelf && id === upd.client.getAuthState().userId) || indexId.has(id)
}
}

10
packages/dispatcher/src/filters/user.ts
View file

@ -94,8 +94,9 @@ export const userId: {
case 'user_typing': {
const id = upd.userId
return (matchSelf && id === upd.client.getAuthState().userId) ||
indexId.has(id)
throw new Error('TODO')
// return (matchSelf && id === upd.client.getAuthState().userId) ||
// indexId.has(id)
}
case 'poll_vote':
case 'story':
@ -110,8 +111,9 @@ export const userId: {
case 'history_read': {
const id = upd.chatId
return (matchSelf && id === upd.client.getAuthState().userId) ||
indexId.has(id)
throw new Error('TODO')
// return (matchSelf && id === upd.client.getAuthState().userId) ||
// indexId.has(id)
}
}

5
packages/file-id/src/convert.ts
View file

@ -1,4 +1,4 @@
import { assertNever, getBasicPeerType, Long, markedPeerIdToBare, tl } from '@mtcute/core'
import { assertNever, Long, parseMarkedPeerId, tl } from '@mtcute/core'
import { parseFileId } from './parse.js'
import { tdFileId as td } from './types.js'
@ -12,8 +12,7 @@ function dialogPhotoToInputPeer(
dialog: td.RawPhotoSizeSourceDialogPhoto | td.RawPhotoSizeSourceDialogPhotoLegacy,
): tl.TypeInputPeer {
const markedPeerId = dialog.id
const peerType = getBasicPeerType(markedPeerId)
const peerId = markedPeerIdToBare(markedPeerId)
const [peerType, peerId] = parseMarkedPeerId(markedPeerId)
if (peerType === 'user') {
return {

2
packages/sqlite/package.json
View file

@ -22,7 +22,7 @@
"dependencies": {
"@mtcute/core": "workspace:^",
"@mtcute/tl-runtime": "workspace:^",
"better-sqlite3": "8.4.0"
"better-sqlite3": "9.2.2"
},
"devDependencies": {
"@mtcute/test": "workspace:^",

169
packages/sqlite/src/driver.ts Normal file
View file

@ -0,0 +1,169 @@
import sqlite3, { Database, Options, Statement } from 'better-sqlite3'
import { BaseStorageDriver, MtUnsupportedError } from '@mtcute/core'
export interface SqliteStorageDriverOptions {
/**
* By default, WAL mode is enabled, which
* significantly improves performance.
* [Learn more](https://github.com/JoshuaWise/better-sqlite3/blob/master/docs/performance.md)
*
* However, you might encounter some issues,
* and if you do, you can disable WAL by passing `true`
*
* @default false
*/
disableWal?: boolean
/**
* Additional options to pass to `better-sqlite3`
*/
options?: Options
}
const MIGRATIONS_TABLE_NAME = 'mtcute_migrations'
const MIGRATIONS_TABLE_SQL = `
create table if not exists ${MIGRATIONS_TABLE_NAME} (
repo text not null primary key,
version integer not null
);
`.trim()
type MigrationFunction = (db: Database) => void
export class SqliteStorageDriver extends BaseStorageDriver {
db!: Database
constructor(
readonly filename = ':memory:',
readonly params?: SqliteStorageDriverOptions,
) {
super()
}
private _pending: [Statement, unknown[]][] = []
private _runMany!: (stmts: [Statement, unknown[]][]) => void
private _migrations: Map<string, Map<number, MigrationFunction>> = new Map()
private _maxVersion: Map<string, number> = new Map()
registerMigration(repo: string, version: number, migration: MigrationFunction): void {
if (this.loaded) {
throw new Error('Cannot register migrations after loading')
}
let map = this._migrations.get(repo)
if (!map) {
map = new Map()
this._migrations.set(repo, map)
}
if (map.has(version)) {
throw new Error(`Migration for ${repo} version ${version} is already registered`)
}
map.set(version, migration)
const prevMax = this._maxVersion.get(repo) ?? 0
if (version > prevMax) {
this._maxVersion.set(repo, version)
}
}
private _onLoad = new Set<(db: Database) => void>()
onLoad(cb: (db: Database) => void): void {
if (this.loaded) {
cb(this.db)
} else {
this._onLoad.add(cb)
}
}
_writeLater(stmt: Statement, params: unknown[]): void {
this._pending.push([stmt, params])
}
_initialize(): void {
const hasLegacyTables = this.db
.prepare("select name from sqlite_master where type = 'table' and name = 'kv'")
.get()
if (hasLegacyTables) {
throw new MtUnsupportedError(
'This database was created with an older version of mtcute, and cannot be used anymore. ' +
'Please delete the database and try again.',
)
}
this.db.exec(MIGRATIONS_TABLE_SQL)
const writeVersion = this.db.prepare(
`insert or replace into ${MIGRATIONS_TABLE_NAME} (repo, version) values (?, ?)`,
)
const getVersion = this.db.prepare(`select version from ${MIGRATIONS_TABLE_NAME} where repo = ?`)
const didUpgrade = new Set<string>()
for (const repo of this._migrations.keys()) {
const res = getVersion.get(repo) as { version: number } | undefined
const startVersion = res?.version ?? 0
let fromVersion = startVersion
const migrations = this._migrations.get(repo)!
const targetVer = this._maxVersion.get(repo)!
while (fromVersion < targetVer) {
const nextVersion = fromVersion + 1
const migration = migrations.get(nextVersion)
if (!migration) {
throw new Error(`No migration for ${repo} to version ${nextVersion}`)
}
migration(this.db)
fromVersion = nextVersion
didUpgrade.add(repo)
}
if (fromVersion !== startVersion) {
writeVersion.run(repo, targetVer)
}
}
}
_load(): void {
this.db = sqlite3(this.filename, {
...this.params?.options,
verbose: this._log.mgr.level >= 5 ? (this._log.verbose as Options['verbose']) : undefined,
})
if (!this.params?.disableWal) {
this.db.pragma('journal_mode = WAL')
}
this._runMany = this.db.transaction((stmts: [Statement, unknown[]][]) => {
stmts.forEach((stmt) => {
stmt[0].run(stmt[1])
})
})
this._initialize()
for (const cb of this._onLoad) cb(this.db)
}
_save(): void {
if (!this._pending.length) return
this._runMany(this._pending)
this._pending = []
}
_destroy(): void {
this.db.close()
}
}

941
packages/sqlite/src/index.ts
View file

@ -1,935 +1,22 @@
// noinspection SqlResolve
import { IMtStorageProvider } from '@mtcute/core'
import sqlite3, { Options } from 'better-sqlite3'
import { SqliteStorageDriver, SqliteStorageDriverOptions } from './driver.js'
import { SqliteAuthKeysRepository } from './repository/auth-keys.js'
import { SqliteKeyValueRepository } from './repository/kv.js'
import { SqlitePeersRepository } from './repository/peers.js'
import { SqliteRefMessagesRepository } from './repository/ref-messages.js'
import { ITelegramStorage, mtp, tl, toggleChannelIdMark } from '@mtcute/core'
import {
beforeExit,
Logger,
longFromFastString,
longToFastString,
LruMap,
throttle,
ThrottledFunction,
TlBinaryReader,
TlBinaryWriter,
TlReaderMap,
TlWriterMap,
} from '@mtcute/core/utils.js'
// todo: add testMode to "self"
function getInputPeer(row: SqliteEntity | ITelegramStorage.PeerInfo): tl.TypeInputPeer {
const id = row.id
switch (row.type) {
case 'user':
return {
_: 'inputPeerUser',
userId: id,
accessHash: 'accessHash' in row ? row.accessHash : longFromFastString(row.hash),
}
case 'chat':
return {
_: 'inputPeerChat',
chatId: -id,
}
case 'channel':
return {
_: 'inputPeerChannel',
channelId: toggleChannelIdMark(id),
accessHash: 'accessHash' in row ? row.accessHash : longFromFastString(row.hash),
}
}
throw new Error(`Invalid peer type: ${row.type}`)
}
const CURRENT_VERSION = 5
// language=SQLite format=false
const TEMP_AUTH_TABLE = `
create table temp_auth_keys (
dc integer not null,
idx integer not null,
key blob not null,
expires integer not null,
primary key (dc, idx)
);
`
// language=SQLite format=false
const MESSAGE_REFS_TABLE = `
create table message_refs (
peer_id integer primary key,
chat_id integer not null,
msg_id integer not null
);
create index idx_message_refs on message_refs (chat_id, msg_id);
`
// language=SQLite format=false
const SCHEMA = `
create table kv (
key text primary key,
value text not null
);
create table state (
key text primary key,
value text not null,
expires number
);
create table auth_keys (
dc integer primary key,
key blob not null
);
${TEMP_AUTH_TABLE}
create table pts (
channel_id integer primary key,
pts integer not null
);
create table entities (
id integer primary key,
hash text not null,
type text not null,
username text,
phone text,
updated integer not null,
"full" blob
);
create index idx_entities_username on entities (username);
create index idx_entities_phone on entities (phone);
${MESSAGE_REFS_TABLE}
`
// language=SQLite format=false
const RESET = `
delete from kv where key <> 'ver';
delete from state;
delete from pts;
delete from entities;
delete from message_refs;
`
const RESET_AUTH_KEYS = `
delete from auth_keys;
delete from temp_auth_keys;
`
const USERNAME_TTL = 86400000 // 24 hours
interface SqliteEntity {
id: number
hash: string
type: string
username?: string
phone?: string
updated: number
full: Uint8Array
}
interface CacheItem {
peer: tl.TypeInputPeer
full: tl.TypeUser | tl.TypeChat | null
}
interface FsmItem<T = unknown> {
value: T
expires?: number
}
interface MessageRef {
peer_id: number
chat_id: number
msg_id: number
}
const STATEMENTS = {
getKv: 'select value from kv where key = ?',
setKv: 'insert or replace into kv (key, value) values (?, ?)',
delKv: 'delete from kv where key = ?',
getState: 'select value, expires from state where key = ?',
setState: 'insert or replace into state (key, value, expires) values (?, ?, ?)',
delState: 'delete from state where key = ?',
getAuth: 'select key from auth_keys where dc = ?',
getAuthTemp: 'select key from temp_auth_keys where dc = ? and idx = ? and expires > ?',
setAuth: 'insert or replace into auth_keys (dc, key) values (?, ?)',
setAuthTemp: 'insert or replace into temp_auth_keys (dc, idx, key, expires) values (?, ?, ?, ?)',
delAuth: 'delete from auth_keys where dc = ?',
delAuthTemp: 'delete from temp_auth_keys where dc = ? and idx = ?',
delAllAuthTemp: 'delete from temp_auth_keys where dc = ?',
getPts: 'select pts from pts where channel_id = ?',
setPts: 'insert or replace into pts (channel_id, pts) values (?, ?)',
updateUpdated: 'update entities set updated = ? where id = ?',
updateCachedEnt: 'update entities set username = ?, phone = ?, updated = ?, "full" = ? where id = ?',
upsertEnt:
'insert or replace into entities (id, hash, type, username, phone, updated, "full") values (?, ?, ?, ?, ?, ?, ?)',
getEntById: 'select * from entities where id = ?',
getEntByPhone: 'select * from entities where phone = ? limit 1',
getEntByUser: 'select * from entities where username = ? limit 1',
storeMessageRef: 'insert or replace into message_refs (peer_id, chat_id, msg_id) values (?, ?, ?)',
getMessageRef: 'select chat_id, msg_id from message_refs where peer_id = ?',
delMessageRefs: 'delete from message_refs where chat_id = ? and msg_id = ?',
delAllMessageRefs: 'delete from message_refs where peer_id = ?',
delStaleState: 'delete from state where expires < ?',
} as const
const EMPTY_BUFFER = new Uint8Array(0)
/**
* SQLite backed storage for mtcute.
*
* Uses `better-sqlite3` library
*/
export class SqliteStorage implements ITelegramStorage /*, IStateStorage*/ {
private _db!: sqlite3.Database
private _statements!: Record<keyof typeof STATEMENTS, sqlite3.Statement>
private readonly _filename: string
private _pending: [sqlite3.Statement, unknown[]][] = []
private _pendingUnimportant: Record<number, unknown[]> = {}
private _cache?: LruMap<number, CacheItem>
private _fsmCache?: LruMap<string, FsmItem>
private _rlCache?: LruMap<string, FsmItem>
private _wal?: boolean
private _reader!: TlBinaryReader
private _saveUnimportantLater: ThrottledFunction
private _vacuumTimeout?: NodeJS.Timeout
private _vacuumInterval: number
private _cleanupUnregister?: () => void
private log!: Logger
private readerMap!: TlReaderMap
private writerMap!: TlWriterMap
/**
* @param filename Database file name, or `:memory:` for in-memory DB
* @param params
*/
export class SqliteStorage implements IMtStorageProvider {
constructor(
filename = ':memory:',
params?: {
/**
* Entities cache size, in number of entities.
*
* Recently encountered entities are cached in memory,
* to avoid redundant database calls. Set to 0 to
* disable caching (not recommended)
*
* Note that by design in-memory cached is only
* used when finding peer by ID, since other
* kinds of lookups (phone, username) may get stale quickly
*
* @default `100`
*/
cacheSize?: number
/**
* FSM states cache size, in number of keys.
*
* Recently created/fetched FSM states are cached
* in memory to avoid redundant database calls.
* If you are having problems with this (e.g. stale
* state in case of concurrent accesses), you
* can disable this by passing `0`
*
* @default `100`
*/
fsmCacheSize?: number
/**
* Rate limit states cache size, in number of keys.
*
* Recently created/used rate limits are cached
* in memory to avoid redundant database calls.
* If you are having problems with this (e.g. stale
* state in case of concurrent accesses), you
* can disable this by passing `0`
*
* @default `100`
*/
rlCacheSize?: number
/**
* By default, WAL mode is enabled, which
* significantly improves performance.
* [Learn more](https://github.com/JoshuaWise/better-sqlite3/blob/master/docs/performance.md)
*
* However, you might encounter some issues,
* and if you do, you can disable WAL by passing `true`
*
* @default false
*/
disableWal?: boolean
/**
* Updates to already cached in-memory entities are only
* applied in DB once in a while, to avoid redundant
* DB calls.
*
* If you are having issues with this, you can set this to `0`
*
* @default `30000` (30 sec)
*/
unimportantSavesDelay?: number
/**
* Interval in milliseconds for vacuuming the storage.
*
* When vacuuming, the storage will remove expired FSM
* states to reduce disk and memory usage.
*
* @default `300_000` (5 minutes)
*/
vacuumInterval?: number
/**
* Whether to finalize database before exiting.
*
* @default `true`
*/
cleanup?: boolean
},
readonly filename = ':memory:',
readonly params?: SqliteStorageDriverOptions,
) {
this._filename = filename
if (params?.cacheSize !== 0) {
this._cache = new LruMap(params?.cacheSize ?? 100)
}
if (params?.fsmCacheSize !== 0) {
this._fsmCache = new LruMap(params?.fsmCacheSize ?? 100)
}
if (params?.rlCacheSize !== 0) {
this._rlCache = new LruMap(params?.rlCacheSize ?? 100)
}
this._wal = !params?.disableWal
this._saveUnimportant = this._saveUnimportant.bind(this)
this._saveUnimportantLater = throttle(this._saveUnimportant, params?.unimportantSavesDelay ?? 30000)
this._vacuumInterval = params?.vacuumInterval ?? 300_000
if (params?.cleanup !== false) {
this._cleanupUnregister = beforeExit(() => this._destroy())
}
}
setup(log: Logger, readerMap: TlReaderMap, writerMap: TlWriterMap): void {
this.log = log.create('sqlite')
this.readerMap = readerMap
this.writerMap = writerMap
this._reader = new TlBinaryReader(readerMap, EMPTY_BUFFER)
}
private _readFullPeer(data: Uint8Array): tl.TypeUser | tl.TypeChat | null {
this._reader = new TlBinaryReader(this.readerMap, data)
let obj
try {
obj = this._reader.object()
} catch (e) {
// object might be from an older tl layer, in which case
// it should be ignored (i guess?????)
obj = null
}
return obj as tl.TypeUser | tl.TypeChat | null
}
private _addToCache(id: number, item: CacheItem): void {
if (this._cache) {
this._cache.set(id, item)
}
}
private _getFromKv<T>(key: string): T | null {
const row = this._statements.getKv.get(key) as { value: string } | null
return row ? (JSON.parse(row.value) as T) : null
}
private _setToKv(key: string, value: unknown, now = false): void {
const query = value === null ? this._statements.delKv : this._statements.setKv
const params = value === null ? [key] : [key, JSON.stringify(value)]
if (now) {
query.run(params)
} else {
this._pending.push([query, params])
}
}
private _runMany!: (stmts: [sqlite3.Statement, unknown[]][]) => void
private _updateManyPeers!: (updates: unknown[][]) => void
private _upgradeDatabase(from: number): void {
if (from < 2 || from > CURRENT_VERSION) {
// 1 version was skipped during development
// yes i am too lazy to make auto-migrations for them
throw new Error('Unsupported session version, please migrate manually')
}
if (from === 2) {
// PFS support added
this._db.exec(TEMP_AUTH_TABLE)
from = 3
}
if (from === 3) {
// media dc support added
const oldDc = this._db.prepare("select value from kv where key = 'def_dc'").get()
if (oldDc) {
const oldDcValue = JSON.parse((oldDc as { value: string }).value) as tl.RawDcOption
this._db.prepare("update kv set value = ? where key = 'def_dc'").run([
JSON.stringify({
main: oldDcValue,
media: oldDcValue,
}),
])
}
from = 4
}
if (from === 4) {
// message references support added
this._db.exec(MESSAGE_REFS_TABLE)
from = 5
}
if (from !== CURRENT_VERSION) {
// an assertion just in case i messed up
throw new Error('Migration incomplete')
}
}
private _initializeStatements(): void {
this._statements = {} as unknown as typeof this._statements
Object.entries(STATEMENTS).forEach(([name, sql]) => {
this._statements[name as keyof typeof this._statements] = this._db.prepare(sql)
})
}
private _initialize(): void {
const hasTables = this._db.prepare("select name from sqlite_master where type = 'table' and name = 'kv'").get()
if (hasTables) {
// tables already exist, check version
const versionResult = this._db.prepare("select value from kv where key = 'ver'").get()
const version = Number((versionResult as { value: number }).value)
this.log.debug('current db version = %d', version)
if (version < CURRENT_VERSION) {
this._upgradeDatabase(version)
this._db.prepare("update kv set value = ? where key = 'ver'").run(CURRENT_VERSION)
}
// prepared statements expect latest schema, so we need to upgrade first
this._initializeStatements()
} else {
// create tables
this.log.debug('creating tables, db version = %d', CURRENT_VERSION)
this._db.exec(SCHEMA)
this._initializeStatements()
this._setToKv('ver', CURRENT_VERSION, true)
}
}
private _vacuum(): void {
this._statements.delStaleState.run(Date.now())
// local caches aren't cleared because it would be too expensive
}
load(): void {
this._db = sqlite3(this._filename, {
verbose: this.log.mgr.level >= 5 ? (this.log.verbose as Options['verbose']) : undefined,
})
this._initialize()
// init wal if needed
if (this._wal) {
this._db.pragma('journal_mode = WAL')
}
// helper methods
this._runMany = this._db.transaction((stmts: [sqlite3.Statement, unknown[]][]) => {
stmts.forEach((stmt) => {
stmt[0].run(stmt[1])
})
})
this._updateManyPeers = this._db.transaction((data: unknown[][]) => {
data.forEach((it: unknown) => {
this._statements.updateCachedEnt.run(it)
})
})
clearInterval(this._vacuumTimeout)
this._vacuumTimeout = setInterval(this._vacuum.bind(this), this._vacuumInterval)
}
private _saveUnimportant() {
// unimportant changes are changes about cached in memory entities,
// that don't really need to be cached right away.
// to avoid redundant DB calls, these changes are persisted
// no more than once every 30 seconds.
//
// additionally, to avoid redundant changes that
// are immediately overwritten, we use object instead
// of an array, where the key is marked peer id,
// and value is the arguments array, since
// the query is always `updateCachedEnt`
const items = Object.values(this._pendingUnimportant)
if (!items.length) return
this._updateManyPeers(items)
this._pendingUnimportant = {}
}
save(): void {
if (!this._pending.length) return
this._runMany(this._pending)
this._pending = []
this._saveUnimportantLater()
}
private _destroy() {
this._saveUnimportant()
this._db.close()
clearInterval(this._vacuumTimeout)
this._saveUnimportantLater.reset()
}
destroy(): void {
this._destroy()
this._cleanupUnregister?.()
}
reset(withAuthKeys = false): void {
this._db.exec(RESET)
if (withAuthKeys) this._db.exec(RESET_AUTH_KEYS)
this._pending = []
this._pendingUnimportant = {}
this._cache?.clear()
this._fsmCache?.clear()
this._rlCache?.clear()
this._saveUnimportantLater.reset()
}
setDefaultDcs(dc: ITelegramStorage.DcOptions | null): void {
return this._setToKv('def_dc', dc, true)
}
getDefaultDcs(): ITelegramStorage.DcOptions | null {
return this._getFromKv('def_dc')
}
getFutureSalts(dcId: number): mtp.RawMt_future_salt[] | null {
return (
this._getFromKv<string[]>(`futureSalts:${dcId}`)?.map((it) => {
const [salt, validSince, validUntil] = it.split(',')
return {
_: 'mt_future_salt',
validSince: Number(validSince),
validUntil: Number(validUntil),
salt: longFromFastString(salt),
}
}) ?? null
)
}
setFutureSalts(dcId: number, salts: mtp.RawMt_future_salt[]): void {
return this._setToKv(
`futureSalts:${dcId}`,
salts.map((salt) => `${longToFastString(salt.salt)},${salt.validSince},${salt.validUntil}`),
true,
)
}
getAuthKeyFor(dcId: number, tempIndex?: number): Uint8Array | null {
let row
if (tempIndex !== undefined) {
row = this._statements.getAuthTemp.get(dcId, tempIndex, Date.now())
} else {
row = this._statements.getAuth.get(dcId)
}
return row ? (row as { key: Uint8Array }).key : null
}
setAuthKeyFor(dcId: number, key: Uint8Array | null): void {
if (key !== null) {
this._statements.setAuth.run(dcId, key)
} else {
this._statements.delAuth.run(dcId)
}
}
setTempAuthKeyFor(dcId: number, index: number, key: Uint8Array | null, expires: number): void {
if (key !== null) {
this._statements.setAuthTemp.run(dcId, index, key, expires)
} else {
this._statements.delAuthTemp.run(dcId, index)
}
}
dropAuthKeysFor(dcId: number): void {
this._statements.delAuth.run(dcId)
this._statements.delAllAuthTemp.run(dcId)
}
private _cachedSelf?: ITelegramStorage.SelfInfo | null
getSelf(): ITelegramStorage.SelfInfo | null {
if (this._cachedSelf !== undefined) return this._cachedSelf
const self = this._getFromKv<ITelegramStorage.SelfInfo | null>('self')
this._cachedSelf = self
return self
}
setSelf(self: ITelegramStorage.SelfInfo | null): void {
this._cachedSelf = self
return this._setToKv('self', self, true)
}
getUpdatesState(): [number, number, number, number] | null {
const pts = this._getFromKv<number>('pts')
if (pts == null) return null
return [
pts,
this._getFromKv<number>('qts') ?? 0,
this._getFromKv<number>('date') ?? 0,
this._getFromKv<number>('seq') ?? 0,
]
}
setUpdatesPts(val: number): void {
return this._setToKv('pts', val)
}
setUpdatesQts(val: number): void {
return this._setToKv('qts', val)
}
setUpdatesDate(val: number): void {
return this._setToKv('date', val)
}
setUpdatesSeq(val: number): void {
return this._setToKv('seq', val)
}
getChannelPts(entityId: number): number | null {
const row = this._statements.getPts.get(entityId)
return row ? (row as { pts: number }).pts : null
}
setManyChannelPts(values: Map<number, number>): void {
for (const [cid, pts] of values) {
this._pending.push([this._statements.setPts, [cid, pts]])
}
}
updatePeers(peers: ITelegramStorage.PeerInfo[]): void {
peers.forEach((peer) => {
const cached = this._cache?.get(peer.id)
if (cached && 'accessHash' in cached.peer && cached.peer.accessHash.eq(peer.accessHash)) {
// when entity is cached and hash is the same, an update query is needed,
// since some field in the full entity might have changed, or the username/phone
//
// since it is cached, we know for sure that it already exists in db,
// so we can safely use `update` instead of `insert or replace`
//
// to avoid too many DB calls, and since these updates are pretty common,
// they are grouped and applied in batches no more than once every 30sec (or user-defined).
//
// until then, they are either served from in-memory cache,
// or an older version is fetched from DB
this._pendingUnimportant[peer.id] = [
peer.username,
peer.phone,
Date.now(),
TlBinaryWriter.serializeObject(this.writerMap, peer.full),
peer.id,
]
cached.full = peer.full
} else {
// entity is not cached in memory, or the access hash has changed
// we need to update it in the DB asap, and also update the in-memory cache
this._pending.push([
this._statements.upsertEnt,
[
peer.id,
longToFastString(peer.accessHash),
peer.type,
peer.username,
peer.phone,
Date.now(),
TlBinaryWriter.serializeObject(this.writerMap, peer.full),
],
])
this._addToCache(peer.id, {
peer: getInputPeer(peer),
full: peer.full,
})
// we have the full peer, we no longer need the references
// we can skip this in the other branch, since in that case it would've already been deleted
if (!this._cachedSelf?.isBot) {
this._pending.push([this._statements.delAllMessageRefs, [peer.id]])
}
}
})
}
private _findPeerByReference(peerId: number): tl.TypeInputPeer | null {
const row = this._statements.getMessageRef.get(peerId) as MessageRef | null
if (!row) return null
const chat = this.getPeerById(row.chat_id, false)
if (!chat) return null
if (peerId > 0) {
// user
return {
_: 'inputPeerUserFromMessage',
peer: chat,
userId: peerId,
msgId: row.msg_id,
}
}
// channel
return {
_: 'inputPeerChannelFromMessage',
peer: chat,
channelId: toggleChannelIdMark(peerId),
msgId: row.msg_id,
}
}
getPeerById(peerId: number, allowRefs = true): tl.TypeInputPeer | null {
const cached = this._cache?.get(peerId)
if (cached) return cached.peer
const row = this._statements.getEntById.get(peerId) as SqliteEntity | null
if (row) {
const peer = getInputPeer(row)
this._addToCache(peerId, {
peer,
full: this._readFullPeer(row.full),
})
return peer
}
if (allowRefs) {
return this._findPeerByReference(peerId)
}
return null
}
getPeerByPhone(phone: string): tl.TypeInputPeer | null {
const row = this._statements.getEntByPhone.get(phone) as SqliteEntity | null
if (row) {
const peer = getInputPeer(row)
this._addToCache(row.id, {
peer,
full: this._readFullPeer(row.full),
})
return peer
}
return null
}
getPeerByUsername(username: string): tl.TypeInputPeer | null {
const row = this._statements.getEntByUser.get(username.toLowerCase()) as SqliteEntity | null
if (!row || Date.now() - row.updated > USERNAME_TTL) return null
if (row) {
const peer = getInputPeer(row)
this._addToCache(row.id, {
peer,
full: this._readFullPeer(row.full),
})
return peer
}
return null
}
getFullPeerById(id: number): tl.TypeUser | tl.TypeChat | null {
const cached = this._cache?.get(id)
if (cached) return cached.full
const row = this._statements.getEntById.get(id) as SqliteEntity | null
if (row) {
const full = this._readFullPeer(row.full)
this._addToCache(id, {
peer: getInputPeer(row),
full,
})
return full
}
return null
}
saveReferenceMessage(peerId: number, chatId: number, messageId: number): void {
this._pending.push([this._statements.storeMessageRef, [peerId, chatId, messageId]])
}
deleteReferenceMessages(chatId: number, messageIds: number[]): void {
for (const id of messageIds) {
this._pending.push([this._statements.delMessageRefs, [chatId, id]])
}
}
// IStateStorage implementation
getState(key: string, parse = true): unknown {
let val: FsmItem | undefined = this._fsmCache?.get(key)
const cached = val
if (!val) {
val = this._statements.getState.get(key) as FsmItem | undefined
if (val && parse) {
val.value = JSON.parse(val.value as string)
}
}
if (!val) return null
if (val.expires && val.expires < Date.now()) {
// expired
if (cached) {
// hot path. if it's cached, then cache is definitely enabled
this._fsmCache!.delete(key)
}
this._statements.delState.run(key)
return null
}
return val.value
}
setState(key: string, state: unknown, ttl?: number, parse = true): void {
const item: FsmItem = {
value: state,
expires: ttl ? Date.now() + ttl * 1000 : undefined,
}
this._fsmCache?.set(key, item)
this._statements.setState.run(key, parse ? JSON.stringify(item.value) : item.value, item.expires)
}
deleteState(key: string): void {
this._fsmCache?.delete(key)
this._statements.delState.run(key)
}
getCurrentScene(key: string): string | null {
return this.getState(`$current_scene_${key}`, false) as string | null
}
setCurrentScene(key: string, scene: string, ttl?: number): void {
return this.setState(`$current_scene_${key}`, scene, ttl, false)
}
deleteCurrentScene(key: string): void {
this.deleteState(`$current_scene_${key}`)
}
getRateLimit(key: string, limit: number, window: number): [number, number] {
// leaky bucket
const now = Date.now()
let val = this._rlCache?.get(key) as FsmItem<number> | undefined
const cached = val
if (!val) {
const got = this._statements.getState.get(`$rate_limit_${key}`)
if (got) {
val = got as FsmItem<number>
}
}
// hot path. rate limit fsm entries always have an expiration date
readonly driver = new SqliteStorageDriver(this.filename, this.params)
if (!val || val.expires! < now) {
// expired or does not exist
const item: FsmItem<number> = {
expires: now + window * 1000,
value: limit,
}
this._statements.setState.run(`$rate_limit_${key}`, item.value, item.expires)
this._rlCache?.set(key, item)
return [item.value, item.expires!]
}
if (val.value > 0) {
val.value -= 1
this._statements.setState.run(`$rate_limit_${key}`, val.value, val.expires)
if (!cached) {
// add to cache
// if cached, cache is updated since `val === cached`
this._rlCache?.set(key, val)
}
}
return [val.value, val.expires!]
}
resetRateLimit(key: string): void {
this._rlCache?.delete(key)
this._statements.delState.run(`$rate_limit_${key}`)
}
readonly authKeys = new SqliteAuthKeysRepository(this.driver)
readonly kv = new SqliteKeyValueRepository(this.driver)
readonly refMessages = new SqliteRefMessagesRepository(this.driver)
readonly peers = new SqlitePeersRepository(this.driver)
}

98
packages/sqlite/src/repository/auth-keys.ts Normal file
View file

@ -0,0 +1,98 @@
import { Statement } from 'better-sqlite3'
import { IAuthKeysRepository } from '@mtcute/core/src/storage/repository/auth-keys.js'
import { SqliteStorageDriver } from '../driver.js'
interface AuthKeyDto {
dc: number
key: Uint8Array
}
interface TempAuthKeyDto extends AuthKeyDto {
expires?: number
idx?: number
}
export class SqliteAuthKeysRepository implements IAuthKeysRepository {
constructor(readonly _driver: SqliteStorageDriver) {
_driver.registerMigration('auth_keys', 1, (db) => {
db.exec(`
create table auth_keys (
dc integer primary key,
key blob not null
);
create table temp_auth_keys (
dc integer not null,
idx integer not null,
key blob not null,
expires integer not null,
primary key (dc, idx)
);
`)
})
_driver.onLoad((db) => {
this._get = db.prepare('select key from auth_keys where dc = ?')
this._getTemp = db.prepare('select key from temp_auth_keys where dc = ? and idx = ? and expires > ?')
this._set = db.prepare('insert or replace into auth_keys (dc, key) values (?, ?)')
this._setTemp = this._driver.db.prepare('insert or replace into temp_auth_keys (dc, idx, key, expires) values (?, ?, ?, ?)')
this._del = db.prepare('delete from auth_keys where dc = ?')
this._delTemp = db.prepare('delete from temp_auth_keys where dc = ? and idx = ?')
this._delTempAll = db.prepare('delete from temp_auth_keys where dc = ?')
this._delAll = db.prepare('delete from auth_keys')
})
}
private _set!: Statement
private _del!: Statement
set(dc: number, key: Uint8Array | null): void {
if (!key) {
this._del.run(dc)
return
}
this._set.run(dc, key)
}
private _get!: Statement
get(dc: number): Uint8Array | null {
const row = this._get.get(dc)
if (!row) return null
return (row as AuthKeyDto).key
}
private _setTemp!: Statement
private _delTemp!: Statement
setTemp(dc: number, idx: number, key: Uint8Array | null, expires: number): void {
if (!key) {
this._delTemp.run(dc, idx)
return
}
this._setTemp.run(dc, idx, key, expires)
}
private _getTemp!: Statement
getTemp(dc: number, idx: number, now: number): Uint8Array | null {
const row = this._getTemp.get(dc, idx, now)
if (!row) return null
return (row as TempAuthKeyDto).key
}
private _delTempAll!: Statement
deleteByDc(dc: number): void {
this._del.run(dc)
this._delTempAll.run(dc)
}
private _delAll!: Statement
deleteAll(): void {
this._delAll.run()
}
}

52
packages/sqlite/src/repository/kv.ts Normal file
View file

@ -0,0 +1,52 @@
import { Statement } from 'better-sqlite3'
import { IKeyValueRepository } from '@mtcute/core/src/storage/repository/key-value.js'
import { SqliteStorageDriver } from '../driver.js'
interface KeyValueDto {
key: string
value: Uint8Array
}
export class SqliteKeyValueRepository implements IKeyValueRepository {
constructor(readonly _driver: SqliteStorageDriver) {
_driver.registerMigration('kv', 1, (db) => {
db.exec(`
create table key_value (
key text primary key,
value blob not null
);
`)
})
_driver.onLoad((db) => {
this._get = db.prepare('select value from key_value where key = ?')
this._set = db.prepare('insert or replace into key_value (key, value) values (?, ?)')
this._del = db.prepare('delete from key_value where key = ?')
this._delAll = db.prepare('delete from key_value')
})
}
private _set!: Statement
set(key: string, value: Uint8Array): void {
this._driver._writeLater(this._set, [key, value])
}
private _get!: Statement
get(key: string): Uint8Array | null {
const res = this._get.get(key)
if (!res) return null
return (res as KeyValueDto).value
}
private _del!: Statement
delete(key: string): void {
this._del.run(key)
}
private _delAll!: Statement
deleteAll(): void {
this._delAll.run()
}
}

98
packages/sqlite/src/repository/peers.ts Normal file
View file

@ -0,0 +1,98 @@
import { Statement } from 'better-sqlite3'
import { IPeersRepository } from '@mtcute/core/src/storage/repository/peers.js'
import { SqliteStorageDriver } from '../driver.js'
interface PeerDto {
id: number
hash: string
usernames: string
updated: number
phone: string | null
// eslint-disable-next-line no-restricted-globals
complete: Buffer
}
function mapPeerDto(dto: PeerDto): IPeersRepository.PeerInfo {
return {
id: dto.id,
accessHash: dto.hash,
usernames: JSON.parse(dto.usernames),
updated: dto.updated,
phone: dto.phone || undefined,
complete: dto.complete,
}
}
export class SqlitePeersRepository implements IPeersRepository {
constructor(readonly _driver: SqliteStorageDriver) {
_driver.registerMigration('peers', 1, (db) => {
db.exec(`
create table peers (
id integer primary key,
hash text not null,
usernames json not null,
updated integer not null,
phone text,
complete blob
);
create index idx_peers_usernames on peers (usernames);
create index idx_peers_phone on peers (phone);
`)
})
_driver.onLoad((db) => {
this._store = db.prepare(
'insert or replace into peers (id, hash, usernames, updated, phone, complete) values (?, ?, ?, ?, ?, ?)',
)
this._getById = db.prepare('select * from peers where id = ?')
this._getByUsername = db.prepare('select * from peers where exists (select 1 from json_each(usernames) where value = ?)')
this._getByPhone = db.prepare('select * from peers where phone = ?')
this._delAll = db.prepare('delete from peers')
})
}
private _store!: Statement
store(peer: IPeersRepository.PeerInfo): void {
this._driver._writeLater(this._store, [
peer.id,
peer.accessHash,
// add commas to make it easier to search with LIKE
JSON.stringify(peer.usernames),
peer.updated,
peer.phone,
peer.complete,
])
}
private _getById!: Statement
getById(id: number): IPeersRepository.PeerInfo | null {
const row = this._getById.get(id)
if (!row) return null
return mapPeerDto(row as PeerDto)
}
private _getByUsername!: Statement
getByUsername(username: string): IPeersRepository.PeerInfo | null {
const row = this._getByUsername.get(username)
if (!row) return null
return mapPeerDto(row as PeerDto)
}
private _getByPhone!: Statement
getByPhone(phone: string): IPeersRepository.PeerInfo | null {
const row = this._getByPhone.get(phone)
if (!row) return null
return mapPeerDto(row as PeerDto)
}
private _delAll!: Statement
deleteAll(): void {
this._delAll.run()
}
}

68
packages/sqlite/src/repository/ref-messages.ts Normal file
View file

@ -0,0 +1,68 @@
import { Statement } from 'better-sqlite3'
import { IReferenceMessagesRepository } from '@mtcute/core/src/storage/repository/ref-messages.js'
import { SqliteStorageDriver } from '../driver.js'
interface ReferenceMessageDto {
peer_id: number
chat_id: number
msg_id: number
}
export class SqliteRefMessagesRepository implements IReferenceMessagesRepository {
constructor(readonly _driver: SqliteStorageDriver) {
_driver.registerMigration('ref_messages', 1, (db) => {
db.exec(`
create table message_refs (
peer_id integer not null,
chat_id integer not null,
msg_id integer not null
);
create index idx_message_refs_peer on message_refs (peer_id);
create index idx_message_refs on message_refs (chat_id, msg_id);
`)
})
_driver.onLoad(() => {
this._store = this._driver.db.prepare('insert or replace into message_refs (peer_id, chat_id, msg_id) values (?, ?, ?)')
this._getByPeer = this._driver.db.prepare('select chat_id, msg_id from message_refs where peer_id = ?')
this._del = this._driver.db.prepare('delete from message_refs where chat_id = ? and msg_id = ?')
this._delByPeer = this._driver.db.prepare('delete from message_refs where peer_id = ?')
this._delAll = this._driver.db.prepare('delete from message_refs')
})
}
private _store!: Statement
store(peerId: number, chatId: number, msgId: number): void {
this._store.run(peerId, chatId, msgId)
}
private _getByPeer!: Statement
getByPeer(peerId: number): [number, number] | null {
const res = this._getByPeer.get(peerId)
if (!res) return null
const res_ = res as ReferenceMessageDto
return [res_.chat_id, res_.msg_id]
}
private _del!: Statement
delete(chatId: number, msgIds: number[]): void {
for (const msgId of msgIds) {
this._driver._writeLater(this._del, [chatId, msgId])
}
}
private _delByPeer!: Statement
deleteByPeer(peerId: number): void {
this._delByPeer.run(peerId)
}
private _delAll!: Statement
deleteAll(): void {
this._delAll.run()
}
}

55
packages/sqlite/test/sqlite.test.ts
View file

@ -1,53 +1,28 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { afterAll, beforeAll, describe } from 'vitest'
import { stubPeerUser, testStateStorage, testStorage } from '@mtcute/test'
import { testAuthKeysRepository } from '@mtcute/core/src/storage/repository/auth-keys.test-utils.js'
import { testKeyValueRepository } from '@mtcute/core/src/storage/repository/key-value.test-utils.js'
import { testPeersRepository } from '@mtcute/core/src/storage/repository/peers.test-utils.js'
import { testRefMessagesRepository } from '@mtcute/core/src/storage/repository/ref-messages.test-utils.js'
import { LogManager } from '@mtcute/core/utils.js'
import { SqliteStorage } from '../src/index.js'
if (import.meta.env.TEST_ENV === 'node') {
describe('SqliteStorage', () => {
testStorage(new SqliteStorage(), {
// sqlite implements "unimportant" updates, which are batched once every 30sec (tested below)
skipEntityOverwrite: true,
customTests: (s) => {
describe('batching', () => {
beforeAll(() => void vi.useFakeTimers())
afterAll(() => void vi.useRealTimers())
const storage = new SqliteStorage(':memory:')
it('should batch entity writes', async () => {
s.updatePeers([stubPeerUser])
s.updatePeers([{ ...stubPeerUser, username: 'test123' }])
s.save()
// eslint-disable-next-line
expect(Object.keys(s['_pendingUnimportant'])).toEqual([String(stubPeerUser.id)])
// not yet updated
expect(s.getPeerByUsername(stubPeerUser.username!)).not.toBeNull()
expect(s.getPeerByUsername('test123')).toBeNull()
await vi.advanceTimersByTimeAsync(30001)
expect(s.getPeerByUsername(stubPeerUser.username!)).toBeNull()
expect(s.getPeerByUsername('test123')).not.toBeNull()
beforeAll(() => {
storage.driver.setup(new LogManager())
storage.driver.load()
})
it('should batch update state writes', () => {
s.setUpdatesPts(123)
s.setUpdatesQts(456)
s.setUpdatesDate(789)
s.setUpdatesSeq(999)
testAuthKeysRepository(storage.authKeys)
testKeyValueRepository(storage.kv, storage.driver)
testPeersRepository(storage.peers, storage.driver)
testRefMessagesRepository(storage.refMessages, storage.driver)
// not yet updated
expect(s.getUpdatesState()).toBeNull()
s.save()
expect(s.getUpdatesState()).toEqual([123, 456, 789, 999])
})
})
},
})
testStateStorage(new SqliteStorage())
afterAll(() => storage.driver.destroy())
})
} else {
describe.skip('SqliteStorage', () => {})

4
packages/test/src/client.ts
View file

@ -36,7 +36,7 @@ export class StubTelegramClient extends BaseTelegramClient {
}
const dcId = transport._currentDc!.id
const key = storage.getAuthKeyFor(dcId)
const key = storage.authKeys.get(dcId)
if (key) {
this._onRawMessage(storage.decryptOutgoingMessage(transport._crypto, data, dcId))
@ -101,7 +101,7 @@ export class StubTelegramClient extends BaseTelegramClient {
this._knownChats.set(peer.id, peer)
}
await this._cachePeersFrom(peer)
await this.storage.peers.updatePeersFrom(peer)
}
}

Some files were not shown because too many files have changed in this diff Show more