diff --git a/packages/core/src/utils/buffer-utils.ts b/packages/core/src/utils/buffer-utils.ts index 4e2008c5..f8f89908 100644 --- a/packages/core/src/utils/buffer-utils.ts +++ b/packages/core/src/utils/buffer-utils.ts @@ -53,67 +53,3 @@ export function cloneBuffer(buf: Buffer, start = 0, end = buf.length): Buffer { buf.copy(ret, 0, start, end) return ret } - -export function parseUrlSafeBase64(str: string): Buffer { - str = str.replace(/-/g, '+').replace(/_/g, '/') - while (str.length % 4) str += '=' - return Buffer.from(str, 'base64') -} - -export function encodeUrlSafeBase64(buf: Buffer): string { - return buf - .toString('base64') - .replace(/\+/g, '-') - .replace(/\//g, '_') - .replace(/=+$/g, '') -} - -// telegram has some cursed RLE which only encodes consecutive \x00 - -export function telegramRleEncode(buf: Buffer): Buffer { - const len = buf.length - const ret: number[] = [] - let count = 0 - - for (let i = 0; i < len; i++) { - const cur = buf[i] - if (cur === 0) { - count += 1 - } else { - if (count > 0) { - ret.push(0, count) - count = 0 - } - - ret.push(cur) - } - } - - if (count > 0) { - ret.push(0, count) - } - - return Buffer.from(ret) -} - -export function telegramRleDecode(buf: Buffer): Buffer { - const len = buf.length - const ret: number[] = [] - let prev = -1 - - for (let i = 0; i < len; i++) { - const cur = buf[i] - if (prev === 0) { - for (let j = 0; j < cur; j++) { - ret.push(prev) - } - prev = -1 - } else { - if (prev !== -1) ret.push(prev) - prev = cur - } - } - - if (prev !== -1) ret.push(prev) - return Buffer.from(ret) -} diff --git a/packages/core/tests/buffer-utils.spec.ts b/packages/core/tests/buffer-utils.spec.ts index 78cc0a4e..ad738a2e 100644 --- a/packages/core/tests/buffer-utils.spec.ts +++ b/packages/core/tests/buffer-utils.spec.ts @@ -3,11 +3,7 @@ import { expect } from 'chai' import { buffersEqual, cloneBuffer, - encodeUrlSafeBase64, - parseUrlSafeBase64, randomBytes, - telegramRleDecode, - telegramRleEncode, xorBuffer, xorBufferInPlace, } from '../src/utils/buffer-utils' @@ -115,82 +111,6 @@ describe('cloneBuffer', () => { }) }) -describe('parseUrlSafeBase64', () => { - it('should parse url-safe base64', () => { - expect(parseUrlSafeBase64('qu7d8aGTeuF6-g').toString('hex')).eq( - 'aaeeddf1a1937ae17afa' - ) - }) - it('should parse normal base64', () => { - expect(parseUrlSafeBase64('qu7d8aGTeuF6+g==').toString('hex')).eq( - 'aaeeddf1a1937ae17afa' - ) - }) -}) - -describe('encodeUrlSafeBase64', () => { - it('should encode to url-safe base64', () => { - expect( - encodeUrlSafeBase64(Buffer.from('aaeeddf1a1937ae17afa', 'hex')) - ).eq('qu7d8aGTeuF6-g') - }) -}) - -describe('telegramRleEncode', () => { - it('should not modify input if there are no \\x00', () => { - expect( - telegramRleEncode(Buffer.from('aaeeff', 'hex')).toString('hex') - ).eq('aaeeff') - }) - - it('should collapse consecutive \\x00', () => { - expect( - telegramRleEncode(Buffer.from('00000000aa', 'hex')).toString('hex') - ).eq('0004aa') - expect( - telegramRleEncode( - Buffer.from('00000000aa000000aa', 'hex') - ).toString('hex') - ).eq('0004aa0003aa') - expect( - telegramRleEncode(Buffer.from('00000000aa0000', 'hex')).toString( - 'hex' - ) - ).eq('0004aa0002') - expect( - telegramRleEncode(Buffer.from('00aa00', 'hex')).toString('hex') - ).eq('0001aa0001') - }) -}) - -describe('telegramRleDecode', () => { - it('should not mofify input if there are no \\x00', () => { - expect( - telegramRleDecode(Buffer.from('aaeeff', 'hex')).toString('hex') - ).eq('aaeeff') - }) - - it('should expand two-byte sequences starting with \\x00', () => { - expect( - telegramRleDecode(Buffer.from('0004aa', 'hex')).toString('hex') - ).eq('00000000aa') - expect( - telegramRleDecode(Buffer.from('0004aa0000', 'hex')).toString('hex') - ).eq('00000000aa') - expect( - telegramRleDecode(Buffer.from('0004aa0003aa', 'hex')).toString( - 'hex' - ) - ).eq('00000000aa000000aa') - expect( - telegramRleDecode(Buffer.from('0004aa0002', 'hex')).toString('hex') - ).eq('00000000aa0000') - expect( - telegramRleDecode(Buffer.from('0001aa0001', 'hex')).toString('hex') - ).eq('00aa00') - }) -}) - // describe('isProbablyPlainText', () => { // it('should return true for buffers only containing printable ascii', () => { // expect( diff --git a/packages/file-id/README.md b/packages/file-id/README.md new file mode 100644 index 00000000..12054fcb --- /dev/null +++ b/packages/file-id/README.md @@ -0,0 +1,39 @@ +# `@mtcute/file-id` + +A package that is used internally by `@mtcute/client` to parse, serialize +and manipulate TDLib and Bot API compatible File IDs, but can also be used +for any other purposes. + +## Contents +This package exports a number of functions, namely: + - `parseFileId()` which parses provided File ID to an object representing its contents + - `toFileId()` which serializes provided object containing file info to a File ID + - `toUniqueFileId()` which serializes provided object containing file info to a Unique File ID + - `fileIdTo*()` which converts a File ID to an input TL object, which can be used + in RPC calls etc. + +This package also exports namespace `tdFileId`, which contains all the types +used by the library + +## Dependencies +This package uses `@mtcute/core` `BinaryReader` and `BinaryWriter` classes to +work with binary streams. Additionally, it depends on `@mtcute/tl` types to +allow type-safe code for conversion functions. + +Note that `@mtcute/core` itself depends on `@mtcute/tl`, which might +redundantly increase your bundle size in case you don't actually use TL types. + +## Acknowledgements +This is basically a port of a portion of TDLib APIs, but greatly +simplified in usage and made to work seamlessly with the rest of the +MTCute APIs. + +This is a list of files from TDLib repository, from which most of the code was taken: + - [td/telegram/files/FileManager.cpp](https://github.com/tdlib/td/blob/master/td/telegram/files/FileManager.cpp) + - [td/telegram/files/FileLocation.hpp](https://github.com/tdlib/td/blob/master/td/telegram/files/FileLocation.hpp) + - [td/telegram/PhotoSizeSource.h](https://github.com/tdlib/td/blob/master/td/telegram/PhotoSizeSource.h) + - [td/telegram/PhotoSizeSource.hpp](https://github.com/tdlib/td/blob/master/td/telegram/PhotoSizeSource.hpp) + - [td/telegram/Version.h](https://github.com/tdlib/td/blob/master/td/telegram/Version.h) + +Additionally, some of the test cases were taken from a similar Python +library, [luckydonald/telegram_file_id](https://github.com/luckydonald/telegram_file_id) diff --git a/packages/file-id/package.json b/packages/file-id/package.json new file mode 100644 index 00000000..675820ec --- /dev/null +++ b/packages/file-id/package.json @@ -0,0 +1,18 @@ +{ + "name": "@mtcute/file-id", + "private": true, + "version": "0.0.0", + "description": "Support for TDLib and Bot API file ID for MTCute", + "author": "Alisa Sireneva ", + "license": "MIT", + "main": "src/index.ts", + "scripts": { + "test": "mocha -r ts-node/register tests/**/*.spec.ts", + "docs": "npx typedoc", + "build": "tsc" + }, + "dependencies": { + "@mtcute/tl": "^0.0.0", + "@mtcute/core": "^0.0.0" + } +} diff --git a/packages/file-id/src/convert.ts b/packages/file-id/src/convert.ts new file mode 100644 index 00000000..ae3bc593 --- /dev/null +++ b/packages/file-id/src/convert.ts @@ -0,0 +1,263 @@ +import { tl } from '@mtcute/tl' +import { tdFileId, tdFileId as td } from './types' +import { parseFileId } from './parse' +import { getBasicPeerType, markedPeerIdToBare } from '@mtcute/core' +import FileType = tdFileId.FileType + +type FileId = td.RawFullRemoteFileLocation + +function dialogPhotoToInputPeer( + dialog: td.RawPhotoSizeSourceDialogPhoto +): tl.TypeInputPeer { + const markedPeerId = dialog.id.toJSNumber() + const peerType = getBasicPeerType(markedPeerId) + const peerId = markedPeerIdToBare(markedPeerId) + + if (peerType === 'user') { + return { + _: 'inputPeerUser', + userId: peerId, + accessHash: dialog.accessHash, + } + } else if (peerType === 'chat') { + return { + _: 'inputPeerChat', + chatId: peerId, + } + } else { + return { + _: 'inputPeerChannel', + channelId: peerId, + accessHash: dialog.accessHash, + } + } +} + +/** + * Convert a file ID or {@link tdFileId.RawFullRemoteFileLocation} + * to TL object `inputWebFileLocation` + * + * @param fileId File ID, either parsed or as a string + */ +export function fileIdToInputWebFileLocation( + fileId: string | FileId +): tl.RawInputWebFileLocation { + if (typeof fileId === 'string') fileId = parseFileId(fileId) + if (fileId.location._ !== 'web') + throw new td.ConversionError('inputWebFileLocation') + + return { + _: 'inputWebFileLocation', + url: fileId.location.url, + accessHash: fileId.location.accessHash, + } +} + +/** + * Convert a file ID or {@link tdFileId.RawFullRemoteFileLocation} + * to TL object representing an `InputFileLocation` + * + * @param fileId File ID, either parsed or as a string + */ +export function fileIdToInputFileLocation( + fileId: string | FileId +): tl.TypeInputFileLocation { + if (typeof fileId === 'string') fileId = parseFileId(fileId) + + const loc = fileId.location + switch (loc._) { + case 'web': + throw new td.ConversionError('InputFileLocation') + case 'photo': { + switch (loc.source._) { + case 'legacy': + if (!fileId.fileReference) + throw new td.InvalidFileIdError( + 'Expected legacy photo to have file reference' + ) + + return { + _: 'inputPhotoLegacyFileLocation', + fileReference: fileId.fileReference, + id: loc.id, + accessHash: loc.accessHash, + volumeId: loc.volumeId, + localId: loc.localId, + secret: loc.source.secret, + } + case 'thumbnail': + if (!fileId.fileReference) + throw new td.InvalidFileIdError( + 'Expected thumbnail photo to have file reference' + ) + + if ( + loc.source.fileType !== FileType.Photo && + loc.source.fileType !== FileType.Thumbnail + ) + throw new td.InvalidFileIdError( + 'Expected a thumbnail to have a correct file type' + ) + + return { + _: + loc.source.fileType === FileType.Photo + ? 'inputPhotoFileLocation' + : 'inputDocumentFileLocation', + fileReference: fileId.fileReference, + id: loc.id, + accessHash: loc.accessHash, + thumbSize: loc.source.thumbnailType, + } + case 'dialogPhoto': + return { + _: 'inputPeerPhotoFileLocation', + big: loc.source.big, + peer: dialogPhotoToInputPeer(loc.source), + volumeId: loc.volumeId, + localId: loc.localId, + } + case 'stickerSetThumbnail': + return { + _: 'inputStickerSetThumb', + stickerset: { + _: 'inputStickerSetID', + id: loc.source.id, + accessHash: loc.source.accessHash, + }, + volumeId: loc.volumeId, + localId: loc.localId, + } + } + + throw new td.ConversionError('inputFileLocation') + } + case 'common': { + if (!fileId.fileReference) + throw new td.InvalidFileIdError( + 'Expected common to have file reference' + ) + + if (fileId.type === FileType.Encrypted) { + return { + _: 'inputEncryptedFileLocation', + id: loc.id, + accessHash: loc.accessHash, + } + } else if ( + fileId.type === FileType.Secure || + fileId.type === FileType.SecureRaw + ) { + return { + _: 'inputSecureFileLocation', + id: loc.id, + accessHash: loc.accessHash, + } + } else { + return { + _: 'inputDocumentFileLocation', + fileReference: fileId.fileReference, + id: loc.id, + accessHash: loc.accessHash, + thumbSize: '', + } + } + } + } +} + +/** + * Convert a file ID or {@link tdFileId.RawFullRemoteFileLocation} + * to TL object `inputDocument` + * + * @param fileId File ID, either parsed or as a string + */ +export function fileIdToInputDocument( + fileId: string | FileId +): tl.RawInputDocument { + if (typeof fileId === 'string') fileId = parseFileId(fileId) + if ( + fileId.location._ !== 'common' || + fileId.type === FileType.Secure || + fileId.type === FileType.SecureRaw || + fileId.type === FileType.Encrypted + ) + throw new td.ConversionError('inputDocument') + + if (!fileId.fileReference) + throw new td.InvalidFileIdError( + 'Expected document to have file reference' + ) + + return { + _: 'inputDocument', + fileReference: fileId.fileReference, + id: fileId.location.id, + accessHash: fileId.location.accessHash, + } +} + +/** + * Convert a file ID or {@link tdFileId.RawFullRemoteFileLocation} + * to TL object `inputPhoto` + * + * @param fileId File ID, either parsed or as a string + */ +export function fileIdToInputPhoto(fileId: string | FileId): tl.RawInputPhoto { + if (typeof fileId === 'string') fileId = parseFileId(fileId) + if (fileId.location._ !== 'photo') + throw new td.ConversionError('inputPhoto') + + if (!fileId.fileReference) + throw new td.InvalidFileIdError('Expected photo to have file reference') + + return { + _: 'inputPhoto', + fileReference: fileId.fileReference, + id: fileId.location.id, + accessHash: fileId.location.accessHash, + } +} + +/** + * Convert a file ID or {@link tdFileId.RawFullRemoteFileLocation} + * to TL object `inputEncryptedFile` + * + * @param fileId File ID, either parsed or as a string + */ +export function fileIdToEncryptedFile( + fileId: string | FileId +): tl.RawInputEncryptedFile { + if (typeof fileId === 'string') fileId = parseFileId(fileId) + if (fileId.location._ !== 'common' || fileId.type !== FileType.Encrypted) + throw new td.ConversionError('inputEncryptedFile') + + return { + _: 'inputEncryptedFile', + id: fileId.location.id, + accessHash: fileId.location.accessHash, + } +} + +/** + * Convert a file ID or {@link tdFileId.RawFullRemoteFileLocation} + * to TL object `inputSecureFile` + * + * @param fileId File ID, either parsed or as a string + */ +export function fileIdToSecureFile( + fileId: string | FileId +): tl.RawInputSecureFile { + if (typeof fileId === 'string') fileId = parseFileId(fileId) + if ( + fileId.location._ !== 'common' || + (fileId.type !== FileType.Secure && fileId.type !== FileType.SecureRaw) + ) + throw new td.ConversionError('inputSecureFile') + + return { + _: 'inputSecureFile', + id: fileId.location.id, + accessHash: fileId.location.accessHash, + } +} diff --git a/packages/file-id/src/index.ts b/packages/file-id/src/index.ts new file mode 100644 index 00000000..f744251d --- /dev/null +++ b/packages/file-id/src/index.ts @@ -0,0 +1,5 @@ +export * from './convert' +export * from './parse' +export * from './serialize' +export * from './serialize-unique' +export * from './types' diff --git a/packages/file-id/src/parse.ts b/packages/file-id/src/parse.ts new file mode 100644 index 00000000..9757fb36 --- /dev/null +++ b/packages/file-id/src/parse.ts @@ -0,0 +1,251 @@ +import { parseUrlSafeBase64, telegramRleDecode } from './utils' +import { tdFileId as td } from './types' +import { BinaryReader } from '@mtcute/core' + +function parseWebFileLocation( + reader: BinaryReader +): td.RawWebRemoteFileLocation { + return { + _: 'web', + url: reader.string(), + accessHash: reader.long(), + } +} + +function parsePhotoSizeSource(reader: BinaryReader): td.TypePhotoSizeSource { + const variant = reader.int32() + switch (variant) { + case 0 /* LEGACY */: + return { + _: 'legacy', + secret: reader.long(), + } + case 1 /* THUMBNAIL */: { + const fileType = reader.int32() + if (fileType < 0 || fileType >= td.FileType.Size) + throw new td.UnsupportedError( + `Unsupported file type: ${fileType} (${reader.data.toString( + 'base64' + )})` + ) + + const thumbnailType = reader.int32() + if (thumbnailType < 0 || thumbnailType > 255) { + throw new td.InvalidFileIdError( + `Wrong thumbnail type: ${thumbnailType} (${reader.data.toString( + 'base64' + )})` + ) + } + + return { + _: 'thumbnail', + fileType, + thumbnailType: String.fromCharCode(thumbnailType), + } + } + case 2 /* DIALOG_PHOTO_SMALL */: + case 3 /* DIALOG_PHOTO_BIG */: + return { + _: 'dialogPhoto', + big: variant === 3, + id: reader.long(), + accessHash: reader.long(), + } + case 4 /* STICKERSET_THUMBNAIL */: + return { + _: 'stickerSetThumbnail', + id: reader.long(), + accessHash: reader.long(), + } + default: + throw new td.UnsupportedError( + `Unsupported photo size source ${variant} (${reader.data.toString( + 'base64' + )})` + ) + } +} + +function parsePhotoFileLocation( + reader: BinaryReader, + version: number +): td.RawPhotoRemoteFileLocation { + return { + _: 'photo', + id: reader.long(), + accessHash: reader.long(), + volumeId: reader.long(), + source: + version >= 22 + ? parsePhotoSizeSource(reader) + : { + _: 'legacy', + secret: reader.long(), + }, + localId: reader.int32(), + } +} + +function parseCommonFileLocation( + reader: BinaryReader +): td.RawCommonRemoteFileLocation { + return { + _: 'common', + id: reader.long(), + accessHash: reader.long(), + } +} + +function fromPersistentIdV23( + binary: Buffer, + version: number +): td.RawFullRemoteFileLocation { + if (version < 0 || version > td.CURRENT_VERSION) + throw new td.UnsupportedError( + `Unsupported file ID v3 subversion: ${version} (${binary.toString( + 'base64' + )})` + ) + + binary = telegramRleDecode(binary) + + const reader = new BinaryReader(binary) + + let fileType = reader.int32() + + const isWeb = !!(fileType & td.WEB_LOCATION_FLAG) + const hasFileReference = !!(fileType & td.FILE_REFERENCE_FLAG) + + fileType &= ~td.WEB_LOCATION_FLAG + fileType &= ~td.FILE_REFERENCE_FLAG + + if (fileType < 0 || fileType >= td.FileType.Size) + throw new td.UnsupportedError( + `Unsupported file type: ${fileType} (${binary.toString('base64')})` + ) + + const dcId = reader.int32() + + let fileReference: Buffer | null = null + if (hasFileReference) { + fileReference = reader.bytes() + if (fileReference.length === 1 && fileReference[0] === 0x23 /* # */) { + // "invalid file reference" + // see https://github.com/tdlib/td/blob/ed291840d3a841bb5b49457c88c57e8467e4a5b0/td/telegram/files/FileLocation.h#L32 + fileReference = null + } + } + + let location: td.TypeRemoteFileLocation + if (isWeb) { + location = parseWebFileLocation(reader) + } else { + switch (fileType) { + case td.FileType.Photo: + case td.FileType.ProfilePhoto: + case td.FileType.Thumbnail: + case td.FileType.EncryptedThumbnail: + case td.FileType.Wallpaper: { + // location_type = photo + location = parsePhotoFileLocation(reader, version) + + // validate + switch (location.source._) { + case 'thumbnail': + if ( + location.source.fileType !== fileType || + (fileType !== td.FileType.Photo && + fileType !== td.FileType.Thumbnail && + fileType !== td.FileType.EncryptedThumbnail) + ) { + throw new td.InvalidFileIdError( + 'Invalid FileType in PhotoRemoteFileLocation Thumbnail' + ) + } + break + case 'dialogPhoto': + if (fileType !== td.FileType.ProfilePhoto) { + throw new td.InvalidFileIdError( + 'Invalid FileType in PhotoRemoteFileLocation DialogPhoto' + ) + } + break + case 'stickerSetThumbnail': + if (fileType !== td.FileType.Thumbnail) { + throw new td.InvalidFileIdError( + 'Invalid FileType in PhotoRemoteFileLocation StickerSetThumbnail' + ) + } + break + } + + break + } + case td.FileType.Video: + case td.FileType.VoiceNote: + case td.FileType.Document: + case td.FileType.Sticker: + case td.FileType.Audio: + case td.FileType.Animation: + case td.FileType.Encrypted: + case td.FileType.VideoNote: + case td.FileType.SecureRaw: + case td.FileType.Secure: + case td.FileType.Background: + case td.FileType.DocumentAsFile: { + // location_type = common + location = parseCommonFileLocation(reader) + break + } + default: + throw new td.UnsupportedError( + `Invalid file type: ${fileType} (${binary.toString( + 'base64' + )})` + ) + } + } + + return { + _: 'remoteFileLocation', + dcId, + type: fileType, + fileReference, + location, + } +} + +function fromPersistentIdV2(binary: Buffer) { + return fromPersistentIdV23(binary.slice(0, -1), 0) +} + +function fromPersistentIdV3(binary: Buffer) { + const subversion = binary[binary.length - 2] + return fromPersistentIdV23(binary.slice(0, -2), subversion) +} + +/** + * Parse TDLib and Bot API compatible File ID + * + * @param fileId File ID as a base-64 encoded string or Buffer + */ +export function parseFileId( + fileId: string | Buffer +): td.RawFullRemoteFileLocation { + if (typeof fileId === 'string') fileId = parseUrlSafeBase64(fileId) + + const version = fileId[fileId.length - 1] + + if (version === td.PERSISTENT_ID_VERSION_OLD) { + return fromPersistentIdV2(fileId) + } + + if (version === td.PERSISTENT_ID_VERSION) { + return fromPersistentIdV3(fileId) + } + + throw new td.UnsupportedError( + `Unsupported file ID version: ${version} (${fileId.toString('base64')})` + ) +} diff --git a/packages/file-id/src/serialize-unique.ts b/packages/file-id/src/serialize-unique.ts new file mode 100644 index 00000000..38b1a22f --- /dev/null +++ b/packages/file-id/src/serialize-unique.ts @@ -0,0 +1,83 @@ +import { tdFileId as td } from './types' +import { BinaryWriter } from '@mtcute/core/src/utils/binary/binary-writer' +import { encodeUrlSafeBase64, telegramRleEncode } from './utils' + +/** + * Serialize an object with information about file + * to TDLib and Bot API compatible Unique File ID + * + * Unique File IDs can't be used to download or reuse files, + * but they are globally unique, meaning that the same file will + * have the same unique ID regardless of the user who created + * this ID (unlike normal File IDs, that also contain user-bound + * file access hash) + * + * @param location Information about file location + */ +export function toUniqueFileId( + location: Omit +): string { + let type + if (location.location._ === 'web') { + type = 0 + } else { + switch (location.type) { + case td.FileType.Photo: + case td.FileType.ProfilePhoto: + case td.FileType.Thumbnail: + case td.FileType.EncryptedThumbnail: + case td.FileType.Wallpaper: + type = 1 + break + case td.FileType.Video: + case td.FileType.VoiceNote: + case td.FileType.Document: + case td.FileType.Sticker: + case td.FileType.Audio: + case td.FileType.Animation: + case td.FileType.VideoNote: + case td.FileType.Background: + case td.FileType.DocumentAsFile: + type = 2 + break + case td.FileType.SecureRaw: + case td.FileType.Secure: + type = 3 + break + case td.FileType.Encrypted: + type = 4 + break + case td.FileType.Temp: + type = 5 + break + default: + throw new td.InvalidFileIdError( + `Invalid file type: ${location.type}` + ) + } + } + + let writer: BinaryWriter + if (location.location._ === 'photo') { + writer = BinaryWriter.alloc(16) + writer.int32(type) + writer.long(location.location.volumeId) + writer.int32(location.location.localId) + } else if (location.location._ === 'web') { + writer = BinaryWriter.alloc( + Buffer.byteLength(location.location.url, 'utf-8') + 8 + ) + writer.int32(type) + writer.string(location.location.url) + } else if (location.location._ === 'common') { + writer = BinaryWriter.alloc(12) + writer.int32(type) + writer.long(location.location.id) + } else { + throw new td.UnsupportedError( + `Unique IDs are not supported for ${(location.location as any)._}` + ) + } + + return encodeUrlSafeBase64(telegramRleEncode(writer.result())) +} diff --git a/packages/file-id/src/serialize.ts b/packages/file-id/src/serialize.ts new file mode 100644 index 00000000..93f62a23 --- /dev/null +++ b/packages/file-id/src/serialize.ts @@ -0,0 +1,83 @@ +import { tdFileId as td } from './types' +import { BinaryWriter } from '@mtcute/core/src/utils/binary/binary-writer' +import { encodeUrlSafeBase64, telegramRleEncode } from './utils' + +const SUFFIX = Buffer.from([td.CURRENT_VERSION, td.PERSISTENT_ID_VERSION]) + +/** + * Serialize an object with information about file + * to TDLib and Bot API compatible File ID + * + * @param location Information about file location + */ +export function toFileId( + location: Omit +): string { + const loc = location.location + + let type: number = location.type + if (loc._ === 'web') type |= td.WEB_LOCATION_FLAG + if (location.fileReference) type |= td.FILE_REFERENCE_FLAG + + const writer = BinaryWriter.alloc( + loc._ === 'web' + ? // overhead of the web file id: + // 8-16 bytes header, + // 8 bytes for access hash, + // up to 4 bytes for url + Buffer.byteLength(loc.url, 'utf8') + 32 + : // longest file ids are around 80 bytes, so i guess + // we are safe with allocating 100 bytes + 100 + ) + + writer.int32(type) + writer.int32(location.dcId) + if (location.fileReference) { + writer.bytes(location.fileReference) + } + + switch (loc._) { + case 'web': + writer.string(loc.url) + writer.long(loc.accessHash) + break + case 'photo': + writer.long(loc.id) + writer.long(loc.accessHash) + writer.long(loc.volumeId) + + switch (loc.source._) { + case 'legacy': + writer.int32(0) + writer.long(loc.source.secret) + break + case 'thumbnail': + writer.int32(1) + writer.int32(loc.source.fileType) + writer.int32(loc.source.thumbnailType.charCodeAt(0)) + break + case 'dialogPhoto': + writer.int32(loc.source.big ? 3 : 2) + writer.long(loc.source.id) + writer.long(loc.source.accessHash) + break + case 'stickerSetThumbnail': + writer.int32(4) + writer.long(loc.source.id) + writer.long(loc.source.accessHash) + break + } + + writer.int32(loc.localId) + break + case 'common': + writer.long(loc.id) + writer.long(loc.accessHash) + break + } + + return encodeUrlSafeBase64( + Buffer.concat([telegramRleEncode(writer.result()), SUFFIX]) + ) +} diff --git a/packages/file-id/src/types.ts b/packages/file-id/src/types.ts new file mode 100644 index 00000000..9c246a16 --- /dev/null +++ b/packages/file-id/src/types.ts @@ -0,0 +1,193 @@ +import { BigInteger } from 'big-integer' + +type Long = BigInteger + +export namespace tdFileId { + export const PERSISTENT_ID_VERSION_OLD = 2 + export const PERSISTENT_ID_VERSION = 4 + + export const WEB_LOCATION_FLAG = 1 << 24 + export const FILE_REFERENCE_FLAG = 1 << 25 + + export const CURRENT_VERSION = 31 + + /** + * An error occurred while parsing or serializing a File ID + */ + export class FileIdError extends Error {} + + /** + * A newer version of File ID is provided, which is + * currently not supported by the library. + * + * Feel free to open an issue on Github! + */ + export class UnsupportedError extends FileIdError {} + + /** + * File ID was invalid, meaning that something did not + * add up while parsing the file ID, or the file ID object + * contained invalid data. + */ + export class InvalidFileIdError extends FileIdError {} + + /** + * Provided File ID cannot be converted to that TL object. + */ + export class ConversionError extends FileIdError { + constructor (to: string) { + super(`Cannot convert given File ID to ${to}`) + } + } + + export enum FileType { + Thumbnail, + ProfilePhoto, + Photo, + VoiceNote, + Video, + Document, + Encrypted, + Temp, + Sticker, + Audio, + Animation, + EncryptedThumbnail, + Wallpaper, + VideoNote, + SecureRaw, + Secure, + Background, + DocumentAsFile, + Size, + None, + } + + // naming convention just like in @mtcute/tl + + // additionally, `_` discriminator is used, + // so we can interoperate with normal TL objects + // like InputFile just by checking `_` + + // for nested types, we don't bother with full type name + // for discriminator since it is really only used internally, + // so uniqueness is pretty much guaranteed + + /** + * This photo is a legacy photo that is + * represented simply by a secret number + */ + export interface RawPhotoSizeSourceLegacy { + readonly _: 'legacy' + readonly secret: Long + } + + /** + * This photo is a thumbnail, and its size + * is provided here as a one-letter string + */ + export interface RawPhotoSizeSourceThumbnail { + readonly _: 'thumbnail' + readonly fileType: FileType + readonly thumbnailType: string + } + + /** + * This photo is a profile photo of + * some peer, and their ID and access + * hash are provided here. + */ + export interface RawPhotoSizeSourceDialogPhoto { + readonly _: 'dialogPhoto' + readonly big: boolean + readonly id: Long + readonly accessHash: Long + } + + /** + * This photo is a thumbnail for a a sticker set, + * and set's ID and access hash are provided here + */ + export interface RawPhotoSizeSourceStickerSetThumbnail { + readonly _: 'stickerSetThumbnail' + readonly id: Long + readonly accessHash: Long + } + + export type TypePhotoSizeSource = + | RawPhotoSizeSourceLegacy + | RawPhotoSizeSourceThumbnail + | RawPhotoSizeSourceDialogPhoto + | RawPhotoSizeSourceStickerSetThumbnail + + /** + * An external web file + */ + export interface RawWebRemoteFileLocation { + readonly _: 'web' + readonly url: string + readonly accessHash: Long + } + + /** + * A photo, that, in addition to ID and access + * hash, has its own `source` and detailed + * information about photo location on the + * servers. + */ + export interface RawPhotoRemoteFileLocation { + readonly _: 'photo' + readonly id: Long + readonly accessHash: Long + readonly volumeId: Long + readonly source: TypePhotoSizeSource + readonly localId: number + } + + /** + * A common file that is represented as a pair + * of ID and access hash + */ + export interface RawCommonRemoteFileLocation { + readonly _: 'common' + readonly id: Long + readonly accessHash: Long + } + + export type TypeRemoteFileLocation = + | RawWebRemoteFileLocation + | RawPhotoRemoteFileLocation + | RawCommonRemoteFileLocation + + /** + * An object representing information about + * file location, that was either parsed from + * TDLib compatible File ID, or will be parsed + * to one. + * + * This type is supposed to be an intermediate step + * between TL objects and string file IDs, + * and if you are using `@mtcute/client`, you don't + * really need to care about this type at all. + */ + export interface RawFullRemoteFileLocation { + readonly _: 'remoteFileLocation' + + /** + * DC ID where this file is located + */ + readonly dcId: number + /** + * Type of the file + */ + readonly type: FileType + /** + * File reference (if any) + */ + readonly fileReference: Buffer | null + /** + * Context of the file location + */ + readonly location: TypeRemoteFileLocation + } +} diff --git a/packages/file-id/src/utils.ts b/packages/file-id/src/utils.ts new file mode 100644 index 00000000..fb236c04 --- /dev/null +++ b/packages/file-id/src/utils.ts @@ -0,0 +1,63 @@ +export function parseUrlSafeBase64(str: string): Buffer { + str = str.replace(/-/g, '+').replace(/_/g, '/') + while (str.length % 4) str += '=' + return Buffer.from(str, 'base64') +} + +export function encodeUrlSafeBase64(buf: Buffer): string { + return buf + .toString('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/g, '') +} + +// telegram has some cursed RLE which only encodes consecutive \x00 + +export function telegramRleEncode(buf: Buffer): Buffer { + const len = buf.length + const ret: number[] = [] + let count = 0 + + for (let i = 0; i < len; i++) { + const cur = buf[i] + if (cur === 0) { + count += 1 + } else { + if (count > 0) { + ret.push(0, count) + count = 0 + } + + ret.push(cur) + } + } + + if (count > 0) { + ret.push(0, count) + } + + return Buffer.from(ret) +} + +export function telegramRleDecode(buf: Buffer): Buffer { + const len = buf.length + const ret: number[] = [] + let prev = -1 + + for (let i = 0; i < len; i++) { + const cur = buf[i] + if (prev === 0) { + for (let j = 0; j < cur; j++) { + ret.push(prev) + } + prev = -1 + } else { + if (prev !== -1) ret.push(prev) + prev = cur + } + } + + if (prev !== -1) ret.push(prev) + return Buffer.from(ret) +} diff --git a/packages/file-id/tests/parse.spec.ts b/packages/file-id/tests/parse.spec.ts new file mode 100644 index 00000000..6412bee5 --- /dev/null +++ b/packages/file-id/tests/parse.spec.ts @@ -0,0 +1,178 @@ +import { describe, it } from 'mocha' +import { expect } from 'chai' +import { parseFileId } from '../src/parse' +import { tdFileId as td } from '../src/types' +import bigInt from 'big-integer' + +// test file IDs are partially taken from https://github.com/luckydonald/telegram_file_id + +describe('parsing file ids', () => { + const test = (id: string, expected: td.RawFullRemoteFileLocation) => { + expect(parseFileId(id)).eql(expected) + } + + it('parses common file ids', () => { + test( + 'CAACAgIAAxkBAAEJny9gituz1_V_uSKBUuG_nhtzEtFOeQACXFoAAuCjggfYjw_KAAGSnkgfBA', + { + _: 'remoteFileLocation', + dcId: 2, + fileReference: Buffer.from( + '0100099f2f608adbb3d7f57fb9228152e1bf9e1b7312d14e79', + 'hex' + ), + location: { + _: 'common', + accessHash: bigInt('5232780349138767832'), + id: bigInt('541175087705905756'), + }, + type: td.FileType.Sticker, + } + ) + test( + 'BQACAgIAAxkBAAEJnzNgit00IDsKd07OdSeanwz8osecYAACdAwAAueoWEicaPvNdOYEwB8E', + { + _: 'remoteFileLocation', + dcId: 2, + fileReference: Buffer.from( + '0100099f33608add34203b0a774ece75279a9f0cfca2c79c60', + 'hex' + ), + location: { + _: 'common', + accessHash: bigInt('-4610306729174144868'), + id: bigInt('5213102278772264052'), + }, + type: td.FileType.Document, + } + ) + }) + + it('parses thumbnails file ids', () => { + test( + 'AAMCAgADGQEAAQmfL2CK27PX9X-5IoFS4b-eG3MS0U55AAJcWgAC4KOCB9iPD8oAAZKeSK1c8w4ABAEAB20AA1kCAAIfBA', + { + _: 'remoteFileLocation', + dcId: 2, + fileReference: Buffer.from( + '0100099f2f608adbb3d7f57fb9228152e1bf9e1b7312d14e79', + 'hex' + ), + location: { + _: 'photo', + accessHash: bigInt('5232780349138767832'), + id: bigInt('541175087705905756'), + localId: 601, + source: { + _: 'thumbnail', + fileType: td.FileType.Thumbnail, + thumbnailType: 'm', + }, + volumeId: bigInt('250829997'), + }, + type: td.FileType.Thumbnail, + } + ) + }) + + it('parses profile pictures', () => { + // big + test('AQADAgATqfDdly4AAwMAA4siCOX_____AAhKowIAAR4E', { + _: 'remoteFileLocation', + dcId: 2, + fileReference: null, + location: { + _: 'photo', + accessHash: bigInt.zero, + id: bigInt.zero, + localId: 172874, + source: { + _: 'dialogPhoto', + id: bigInt('-452451701'), + accessHash: bigInt.zero, + big: true, + }, + volumeId: bigInt('200116400297'), + }, + type: td.FileType.ProfilePhoto, + }) + + // small + test('AQADAgATqfDdly4AAwIAA4siCOX_____AAhIowIAAR4E', { + _: 'remoteFileLocation', + dcId: 2, + fileReference: null, + location: { + _: 'photo', + accessHash: bigInt.zero, + id: bigInt.zero, + localId: 172872, + source: { + _: 'dialogPhoto', + id: bigInt('-452451701'), + accessHash: bigInt.zero, + big: false, + }, + volumeId: bigInt('200116400297'), + }, + type: td.FileType.ProfilePhoto, + }) + }) + + it('parses channel pictures', () => { + // big + test('AQADAgATySHBDgAEAwAD0npI3Bb___-wfxjpg7QCPf8pBQABHwQ', { + _: 'remoteFileLocation', + dcId: 2, + fileReference: null, + location: { + _: 'photo', + accessHash: bigInt.zero, + id: bigInt.zero, + localId: 338431, + source: { + _: 'dialogPhoto', + id: bigInt('-1001326609710'), + accessHash: bigInt('4396274664911437744'), + big: true, + }, + volumeId: bigInt('247538121'), + }, + type: td.FileType.ProfilePhoto, + }) + // small + test('AQADAgATySHBDgAEAgAD0npI3Bb___-wfxjpg7QCPf0pBQABHwQ', { + _: 'remoteFileLocation', + dcId: 2, + fileReference: null, + location: { + _: 'photo', + accessHash: bigInt.zero, + id: bigInt.zero, + localId: 338429, + source: { + _: 'dialogPhoto', + id: bigInt('-1001326609710'), + accessHash: bigInt('4396274664911437744'), + big: false, + }, + volumeId: bigInt('247538121'), + }, + type: td.FileType.ProfilePhoto, + }) + }) + + it('parses older short file ids', () => { + test('CAADAQADegAD997LEUiQZafDlhIeAg', { + _: 'remoteFileLocation', + dcId: 1, + fileReference: null, + location: { + _: 'common', + accessHash: bigInt('2166960137789870152'), + id: bigInt('1282363671355326586'), + }, + type: td.FileType.Sticker, + }) + }) +}) diff --git a/packages/file-id/tests/serialize-unique.spec.ts b/packages/file-id/tests/serialize-unique.spec.ts new file mode 100644 index 00000000..2b9272c1 --- /dev/null +++ b/packages/file-id/tests/serialize-unique.spec.ts @@ -0,0 +1,42 @@ +import { describe } from 'mocha' +import { expect } from 'chai' +import { parseFileId } from '../src/parse' +import { toUniqueFileId } from '../src/serialize-unique' + +// test file IDs are partially taken from https://github.com/luckydonald/telegram_file_id + +describe('serializing unique file ids', () => { + const test = (id: string, expected: string) => { + expect(toUniqueFileId(parseFileId(id))).eql(expected) + } + + it('serializes unique ids for old file ids', () => { + test('CAADAQADegAD997LEUiQZafDlhIeAg', 'AgADegAD997LEQ') + }) + + it('serializes unique ids for common file ids', () => { + test( + 'CAACAgEAAx0CVgtngQACAuFfU1GY9wiRG7A7jlIBbP2yvAostAACegAD997LEUiQZafDlhIeGwQ', + 'AgADegAD997LEQ' + ) + test( + 'BQACAgIAAxkBAAEJnzNgit00IDsKd07OdSeanwz8osecYAACdAwAAueoWEicaPvNdOYEwB8E', + 'AgADdAwAAueoWEg' + ) + test( + 'AAMCAgADGQEAAQmfM2CK3TQgOwp3Ts51J5qfDPyix5xgAAJ0DAAC56hYSJxo-8105gTAT_bYoy4AAwEAB20AA0JBAAIfBA', + 'AQADT_bYoy4AA0JBAAI' + ) + test( + 'CAACAgIAAxkBAAEJny9gituz1_V_uSKBUuG_nhtzEtFOeQACXFoAAuCjggfYjw_KAAGSnkgfBA', + 'AgADXFoAAuCjggc' + ) + }) + + it('serializes unique ids for profile pictures', () => { + // big + test('AQADAgATySHBDgAEAwAD0npI3Bb___-wfxjpg7QCPf8pBQABHwQ', 'AQADySHBDgAE_ykFAAE') + // small + test('AQADAgATySHBDgAEAgAD0npI3Bb___-wfxjpg7QCPf0pBQABHwQ', 'AQADySHBDgAE_SkFAAE') + }) +}) diff --git a/packages/file-id/tests/serialize.spec.ts b/packages/file-id/tests/serialize.spec.ts new file mode 100644 index 00000000..0124a2f8 --- /dev/null +++ b/packages/file-id/tests/serialize.spec.ts @@ -0,0 +1,20 @@ +import { describe } from 'mocha' +import { expect } from 'chai' +import { parseFileId, toFileId } from '../src' + +describe('serializing to file ids', () => { + it('serializes previously parsed file ids', () => { + const test = (fileId: string) => { + expect(toFileId(parseFileId(fileId))).eq(fileId) + } + + test('CAACAgIAAxkBAAEJny9gituz1_V_uSKBUuG_nhtzEtFOeQACXFoAAuCjggfYjw_KAAGSnkgfBA') + test('BQACAgIAAxkBAAEJnzNgit00IDsKd07OdSeanwz8osecYAACdAwAAueoWEicaPvNdOYEwB8E') + test('AAMCAgADGQEAAQmfL2CK27PX9X-5IoFS4b-eG3MS0U55AAJcWgAC4KOCB9iPD8oAAZKeSK1c8w4ABAEAB20AA1kCAAIfBA') + test('AQADAgATqfDdly4AAwMAA4siCOX_____AAhKowIAAR8E') + test('AQADAgATqfDdly4AAwIAA4siCOX_____AAhIowIAAR8E') + test('AQADAgATySHBDgAEAwAD0npI3Bb___-wfxjpg7QCPf8pBQABHwQ') + test('AQADAgATySHBDgAEAgAD0npI3Bb___-wfxjpg7QCPf0pBQABHwQ') + test('CAADAQADegAD997LEUiQZafDlhIeHwQ') + }) +}) diff --git a/packages/file-id/tests/utils.spec.ts b/packages/file-id/tests/utils.spec.ts new file mode 100644 index 00000000..44ff9134 --- /dev/null +++ b/packages/file-id/tests/utils.spec.ts @@ -0,0 +1,84 @@ +import { describe, it } from 'mocha' +import { expect } from 'chai' +import { + encodeUrlSafeBase64, + parseUrlSafeBase64, + telegramRleDecode, + telegramRleEncode, +} from '../src/utils' + +describe('parseUrlSafeBase64', () => { + it('should parse url-safe base64', () => { + expect(parseUrlSafeBase64('qu7d8aGTeuF6-g').toString('hex')).eq( + 'aaeeddf1a1937ae17afa' + ) + }) + it('should parse normal base64', () => { + expect(parseUrlSafeBase64('qu7d8aGTeuF6+g==').toString('hex')).eq( + 'aaeeddf1a1937ae17afa' + ) + }) +}) + +describe('encodeUrlSafeBase64', () => { + it('should encode to url-safe base64', () => { + expect( + encodeUrlSafeBase64(Buffer.from('aaeeddf1a1937ae17afa', 'hex')) + ).eq('qu7d8aGTeuF6-g') + }) +}) + +describe('telegramRleEncode', () => { + it('should not modify input if there are no \\x00', () => { + expect( + telegramRleEncode(Buffer.from('aaeeff', 'hex')).toString('hex') + ).eq('aaeeff') + }) + + it('should collapse consecutive \\x00', () => { + expect( + telegramRleEncode(Buffer.from('00000000aa', 'hex')).toString('hex') + ).eq('0004aa') + expect( + telegramRleEncode( + Buffer.from('00000000aa000000aa', 'hex') + ).toString('hex') + ).eq('0004aa0003aa') + expect( + telegramRleEncode(Buffer.from('00000000aa0000', 'hex')).toString( + 'hex' + ) + ).eq('0004aa0002') + expect( + telegramRleEncode(Buffer.from('00aa00', 'hex')).toString('hex') + ).eq('0001aa0001') + }) +}) + +describe('telegramRleDecode', () => { + it('should not mofify input if there are no \\x00', () => { + expect( + telegramRleDecode(Buffer.from('aaeeff', 'hex')).toString('hex') + ).eq('aaeeff') + }) + + it('should expand two-byte sequences starting with \\x00', () => { + expect( + telegramRleDecode(Buffer.from('0004aa', 'hex')).toString('hex') + ).eq('00000000aa') + expect( + telegramRleDecode(Buffer.from('0004aa0000', 'hex')).toString('hex') + ).eq('00000000aa') + expect( + telegramRleDecode(Buffer.from('0004aa0003aa', 'hex')).toString( + 'hex' + ) + ).eq('00000000aa000000aa') + expect( + telegramRleDecode(Buffer.from('0004aa0002', 'hex')).toString('hex') + ).eq('00000000aa0000') + expect( + telegramRleDecode(Buffer.from('0001aa0001', 'hex')).toString('hex') + ).eq('00aa00') + }) +}) diff --git a/packages/file-id/tsconfig.json b/packages/file-id/tsconfig.json new file mode 100644 index 00000000..65628325 --- /dev/null +++ b/packages/file-id/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + }, + "include": [ + "./src" + ], + "typedocOptions": { + "name": "@mtcute/file-id", + "includeVersion": true, + "out": "../../docs/packages/file-id", + "listInvalidSymbolLinks": true, + "excludePrivate": true, + "entryPoints": [ + "./src/index.ts" + ] + } +}