diff --git a/packages/file-id/src/serialize-unique.ts b/packages/file-id/src/serialize-unique.ts index 0b50e84b..2d676f21 100644 --- a/packages/file-id/src/serialize-unique.ts +++ b/packages/file-id/src/serialize-unique.ts @@ -149,7 +149,7 @@ export function toUniqueFileId( } case 'web': writer = TlBinaryWriter.alloc( - {}, + undefined, Buffer.byteLength(inputLocation.url, 'utf-8') + 8, ) writer.int(type) diff --git a/packages/file-id/src/serialize.ts b/packages/file-id/src/serialize.ts index d1e4747c..8610f17d 100644 --- a/packages/file-id/src/serialize.ts +++ b/packages/file-id/src/serialize.ts @@ -21,15 +21,16 @@ export function toFileId( if (loc._ === 'web') type |= td.WEB_LOCATION_FLAG if (location.fileReference) type |= td.FILE_REFERENCE_FLAG + // overhead of the web file id: + // 8-16 bytes header, + // 8 bytes for access hash, + // up to 4 bytes for url + // + // longest file ids are around 80 bytes, so i guess + // we are safe with allocating 100 bytes const writer = TlBinaryWriter.alloc( - {}, - loc._ === 'web' ? // overhead of the web file id: - // 8-16 bytes header, - // 8 bytes for access hash, - // up to 4 bytes for url - Buffer.byteLength(loc.url, 'utf8') + 32 : // longest file ids are around 80 bytes, so i guess - // we are safe with allocating 100 bytes - 100, + undefined, + loc._ === 'web' ? Buffer.byteLength(loc.url, 'utf8') + 32 : 100, ) writer.int(type) diff --git a/packages/tl-runtime/src/writer.ts b/packages/tl-runtime/src/writer.ts index 608fd51d..fd0107cc 100644 --- a/packages/tl-runtime/src/writer.ts +++ b/packages/tl-runtime/src/writer.ts @@ -10,6 +10,7 @@ const TWO_PWR_32_DBL = (1 << 16) * (1 << 16) export type TlWriterMap = Record void> & { // eslint-disable-next-line @typescript-eslint/no-explicit-any _bare?: Record void> + _staticSize: Record } /** @@ -165,7 +166,10 @@ export class TlBinaryWriter { * @param objectMap Writers map * @param size Size of the writer's buffer */ - static alloc(objectMap: TlWriterMap, size: number): TlBinaryWriter { + static alloc( + objectMap: TlWriterMap | undefined, + size: number, + ): TlBinaryWriter { return new TlBinaryWriter(objectMap, Buffer.allocUnsafe(size)) } @@ -202,7 +206,9 @@ export class TlBinaryWriter { knownSize = -1, ): Buffer { if (knownSize === -1) { - knownSize = TlSerializationCounter.countNeededBytes(objectMap, obj) + knownSize = + objectMap._staticSize[obj._] || + TlSerializationCounter.countNeededBytes(objectMap, obj) } const writer = TlBinaryWriter.alloc(objectMap, knownSize) diff --git a/packages/tl-runtime/tests/binary-writer.spec.ts b/packages/tl-runtime/tests/binary-writer.spec.ts index 91e2b4e2..6c4a9564 100644 --- a/packages/tl-runtime/tests/binary-writer.spec.ts +++ b/packages/tl-runtime/tests/binary-writer.spec.ts @@ -10,7 +10,7 @@ describe('TlBinaryWriter', () => { const testSingleMethod = ( size: number, fn: (w: TlBinaryWriter) => void, - map: TlWriterMap = {}, + map?: TlWriterMap, ): string => { const w = TlBinaryWriter.alloc(map, size) fn(w) @@ -126,6 +126,8 @@ describe('TlBinaryWriter', () => { w.uint(0xbebf0c3d) w.vector(w.int, obj.vec) }, + // eslint-disable-next-line + _staticSize: {} as any, } it('should write tg-encoded objects', () => { @@ -218,6 +220,8 @@ describe('TlBinaryWriter', () => { w.bytes(obj.pq) w.vector(w.long, obj.serverPublicKeyFingerprints) }, + // eslint-disable-next-line + _staticSize: {} as any, } const length = diff --git a/packages/tl-utils/src/calculator.ts b/packages/tl-utils/src/calculator.ts new file mode 100644 index 00000000..f285bf05 --- /dev/null +++ b/packages/tl-utils/src/calculator.ts @@ -0,0 +1,96 @@ +import { TlEntry } from './types' + +const PRIMITIVES_SIZES: Record = { + int: 4, + long: 8, + int53: 8, + int128: 16, + int256: 32, + double: 8, + boolFalse: 4, + boolTrue: 4, + bool: 4, + Bool: 4, + '#': 4, +} + +export function calculateStaticSizes( + entries: TlEntry[], +): Record { + const staticSizes: Record = {} + const unionSizes: Record> = {} + let changedInLastIteration = true + + function getUnionStaticSize(name: string): number | null { + const values = Object.values(unionSizes[name] ?? {}) + if (values.length === 0) return null + + const first = values[0] + if (first === null) return null + + for (const value of values) { + if (value !== first) return null + } + + return first + } + + function calculateStaticSize(entry: TlEntry): number | null { + if (entry.generics) return null // definitely not static sized + + let size = 4 // constructor id + + for (const arg of entry.arguments) { + if (arg.typeModifiers?.predicate) { + if (arg.type === 'true') { + continue // zero-size type + } + + // cant be static sized + return null + } + + let unionSize + + if (arg.type in PRIMITIVES_SIZES) { + size += PRIMITIVES_SIZES[arg.type] + } else if (arg.type in staticSizes) { + size += staticSizes[arg.type] - 4 // subtract constructor id + } else if ((unionSize = getUnionStaticSize(arg.type))) { + size += unionSize + } else { + // likely not static sized + return null + } + } + + return size + } + + while (changedInLastIteration) { + changedInLastIteration = false + + for (const entry of entries) { + if (staticSizes[entry.name] !== undefined) continue + + const size = calculateStaticSize(entry) + + if (entry.kind === 'class') { + if (!unionSizes[entry.type]) { + unionSizes[entry.type] = {} + } + + unionSizes[entry.type][entry.name] = size + } + + if (size === null) { + continue + } + + staticSizes[entry.name] = size + changedInLastIteration = true + } + } + + return staticSizes +} diff --git a/packages/tl-utils/src/codegen/writer.ts b/packages/tl-utils/src/codegen/writer.ts index ff1d87ef..2143c0fc 100644 --- a/packages/tl-utils/src/codegen/writer.ts +++ b/packages/tl-utils/src/codegen/writer.ts @@ -1,3 +1,4 @@ +import { calculateStaticSizes } from '../calculator' import { computeConstructorIdFromEntry } from '../ctor-id' import { TL_PRIMITIVES, TlEntry } from '../types' import { snakeToCamel } from './utils' @@ -20,6 +21,11 @@ export interface WriterCodegenOptions { */ includePrelude?: boolean + /** + * Whether to include `_staticSize` field + */ + includeStaticSizes?: boolean + /** * Whether to generate bare writer (without constructor id write) */ @@ -31,6 +37,7 @@ const DEFAULT_OPTIONS: WriterCodegenOptions = { variableName: 'm', includePrelude: true, bare: false, + includeStaticSizes: false, } const TL_WRITER_PRELUDE = @@ -168,7 +175,10 @@ export function generateWriterCodeForTlEntries( entries: TlEntry[], params = DEFAULT_OPTIONS, ): string { - const { includePrelude, variableName } = { ...DEFAULT_OPTIONS, ...params } + const { includePrelude, variableName, includeStaticSizes } = { + ...DEFAULT_OPTIONS, + ...params, + } let ret = '' if (includePrelude) ret += TL_WRITER_PRELUDE @@ -201,7 +211,19 @@ export function generateWriterCodeForTlEntries( bare: true, }) + '\n' }) - ret += '}' + ret += '},\n' + } + + if (includeStaticSizes) { + ret += '_staticSize:{\n' + + const staticSizes = calculateStaticSizes(entries) + + Object.keys(staticSizes).forEach((name) => { + ret += `'${name}':${staticSizes[name]},\n` + }) + + ret += '},\n' } return ret + '}' diff --git a/packages/tl-utils/src/merge.ts b/packages/tl-utils/src/merge.ts index 9329e695..61a3df10 100644 --- a/packages/tl-utils/src/merge.ts +++ b/packages/tl-utils/src/merge.ts @@ -1,5 +1,5 @@ import { computeConstructorIdFromEntry } from './ctor-id' -import { TlEntry, TlFullSchema } from './types' +import { TlArgument, TlEntry, TlFullSchema } from './types' /** * Merge multiple TL entries into a single entry. @@ -18,27 +18,28 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string { typeModifiers: first.typeModifiers, id: first.id, comment: first.comment, - generics: first.generics, - arguments: first.arguments, + generics: first.generics?.map((it) => ({ ...it })), + arguments: first.arguments.map((it) => ({ ...it })), } if (result.id === 0) { result.id = computeConstructorIdFromEntry(result) } - const argsIndex: Record = {} + const argsIndex: Record = {} const flagsLastIndex: Record = {} result.arguments.forEach((arg, idx) => { - argsIndex[arg.name] = true + argsIndex[arg.name] = arg if (arg.type === '#') { flagsLastIndex[arg.name] = idx } - // if (arg.predicate) { - // const flagsField = arg.predicate.split('.')[0] - // flagsLastIndex[flagsField] = idx - // } + + if (arg.typeModifiers?.predicate) { + const flagsField = arg.typeModifiers.predicate.split('.')[0] + flagsLastIndex[flagsField] = idx + } }) for (let i = 1; i < entries.length; i++) { @@ -89,7 +90,7 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string { // targetIdx *must* exist, otherwise ids wouldn't match result.arguments.splice(targetIdx + 1, 0, entryArgument) - argsIndex[entryArgument.name] = true + argsIndex[entryArgument.name] = entryArgument // update last indexes // we also need to update subsequent flags if there are any @@ -98,10 +99,17 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string { flagsLastIndex[flag]++ } }) + + continue } // args exists both in result and current entry // since ctor ids match, it must be the same, so we don't need to check + // we still need to merge comments though + + if (!resultArgument.comment && entryArgument.comment) { + resultArgument.comment = entryArgument.comment + } } } diff --git a/packages/tl-utils/src/types.ts b/packages/tl-utils/src/types.ts index b6067874..c412da85 100644 --- a/packages/tl-utils/src/types.ts +++ b/packages/tl-utils/src/types.ts @@ -426,5 +426,5 @@ export const TL_PRIMITIVES = { Bool: 1, true: 1, null: 1, - Object: true, + Object: 1, } as const diff --git a/packages/tl-utils/tests/calculator.spec.ts b/packages/tl-utils/tests/calculator.spec.ts new file mode 100644 index 00000000..9939b85d --- /dev/null +++ b/packages/tl-utils/tests/calculator.spec.ts @@ -0,0 +1,67 @@ +import { expect } from 'chai' +import { describe, it } from 'mocha' + +import { parseTlToEntries } from '../src' +import { calculateStaticSizes } from '../src/calculator' + +describe('calculateStaticSizes', () => { + const test = (tl: string, expected: object) => { + expect(calculateStaticSizes(parseTlToEntries(tl))).eql(expected) + } + + it('computes for constructors without parameters', () => { + test('auth.logOut = Bool;', { 'auth.logOut': 4 }) + }) + + it('computes for constructors with static parameters', () => { + test( + 'auth.exportAuthorization#e5bfffcd dc_id:int = auth.ExportedAuthorization;', + { 'auth.exportAuthorization': 8 }, + ) + }) + + it('correctly skips true fields', () => { + test( + 'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int = help.PromoData;', + { 'help.promoData': 12 }, + ) + }) + + it('correctly skips constructors with predicated fields', () => { + test( + 'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:Peer psa_type:flags.1?string psa_message:flags.2?string = help.PromoData;', + {}, + ) + }) + + it('correctly skips constructors with non-static fields', () => { + test( + 'help.promoData#8c39793f psa_type:string psa_message:string = help.PromoData;', + {}, + ) + }) + + it('correctly handles static-sized children', () => { + test( + 'peerUser#9db1bc6d user_id:int53 = Peer;\n' + + 'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:peerUser = help.PromoData;', + { + peerUser: 12, + 'help.promoData': 20, + }, + ) + }) + + it('correctly handles static-sized union children', () => { + test( + 'peerUser#9db1bc6d user_id:int53 = Peer;\n' + + 'peerChannel#9db1bc6d channel_id:int53 = Peer;\n' + + 'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:Peer = help.PromoData;', + { + peerUser: 12, + peerChannel: 12, + 'help.promoData': 24, + }, + ) + }) +}) diff --git a/packages/tl-utils/tests/codegen/writer.spec.ts b/packages/tl-utils/tests/codegen/writer.spec.ts index 0712048a..efd4f517 100644 --- a/packages/tl-utils/tests/codegen/writer.spec.ts +++ b/packages/tl-utils/tests/codegen/writer.spec.ts @@ -137,7 +137,7 @@ describe('generateWriterCodeForTlEntry', () => { 'future_salt':function(w,v){w.uint(155834844);w.bytes(h(v,'salt'));}, 'future_salts':function(w,v){w.uint(2924480661);w.vector(m._bare[155834844],h(v,'salts'),1);m._bare[155834844](w,h(v,'current'));}, _bare:{ - 155834844:function(w,v){w.bytes(h(v,'salt'));}, + 155834844:function(w=this,v){w.bytes(h(v,'salt'));}, }}`.replace(/^\s+/gm, ''), ) }) diff --git a/packages/tl-utils/tests/merge.spec.ts b/packages/tl-utils/tests/merge.spec.ts index dea79a0d..00713f3b 100644 --- a/packages/tl-utils/tests/merge.spec.ts +++ b/packages/tl-utils/tests/merge.spec.ts @@ -1,10 +1,14 @@ import { expect } from 'chai' import { describe, it } from 'mocha' -import { mergeTlEntries, mergeTlSchemas } from '../src/merge' -import { parseTlToEntries } from '../src/parse' -import { parseFullTlSchema, writeTlEntriesToString } from '../src/schema' -import { writeTlEntryToString } from '../src/stringify' +import { + mergeTlEntries, + mergeTlSchemas, + parseFullTlSchema, + parseTlToEntries, + writeTlEntriesToString, + writeTlEntryToString, +} from '../src' describe('mergeTlEntries', () => { const test = (tl: string, expected: string) => { @@ -99,10 +103,10 @@ describe('mergeTlSchemas', () => { ['---functions---', 'testMethod = Test;'], ], 0, - 'testClass = Test;', - 'testClass2 = Test;', + 'testClass#5d60a438 = Test;', + 'testClass2#39c5c841 = Test;', '---functions---', - 'testMethod = Test;', + 'testMethod#87d8a7d2 = Test;', ) }) @@ -114,7 +118,7 @@ describe('mergeTlSchemas', () => { ['test foo:flags.0?true bar:flags.0?true = Test;'], ], 0, - 'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;', + 'test#1c173316 bar:flags.0?true foo:flags.0?true = Test;', ) }) @@ -126,7 +130,7 @@ describe('mergeTlSchemas', () => { ['test baz:int = Test;'], ], 0, - 'test foo:int = Test;', + 'test#4f6455cd foo:int = Test;', ) await test( [ @@ -135,7 +139,7 @@ describe('mergeTlSchemas', () => { ['test baz:int = Test;'], ], 1, - 'test foo:int = Test;', + 'test#3e993a74 bar:int = Test;', ) await test( [['test foo:int = Test;'], [], ['test bar:int = Test;']], @@ -156,7 +160,7 @@ describe('mergeTlSchemas', () => { ], 0, '// @description test ctor', - 'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;', + 'test#1c173316 bar:flags.0?true foo:flags.0?true = Test;', ) }) @@ -164,13 +168,17 @@ describe('mergeTlSchemas', () => { await test( [ ['test foo:flags.0?true = Test;'], - ['// @bar bar comment', 'test bar:flags.0?true = Test;'], [ - '// @foo foo comment', + '// @description test @bar bar comment', + 'test bar:flags.0?true = Test;', + ], + [ + '// @description test @foo foo comment', 'test foo:flags.0?true bar:flags.0?true = Test;', ], ], 0, + '// @description test', '// @foo foo comment', '// @bar bar comment', 'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;', diff --git a/packages/tl/binary/writer.d.ts b/packages/tl/binary/writer.d.ts index 71542725..16f7910e 100644 --- a/packages/tl/binary/writer.d.ts +++ b/packages/tl/binary/writer.d.ts @@ -1,5 +1,6 @@ declare const __tlWriterMap: Record void> & { _bare: Record unknown> + _staticSize: Record } // eslint-disable-next-line import/no-default-export export default __tlWriterMap diff --git a/packages/tl/scripts/gen-code.ts b/packages/tl/scripts/gen-code.ts index d53ad6b4..760880c3 100644 --- a/packages/tl/scripts/gen-code.ts +++ b/packages/tl/scripts/gen-code.ts @@ -77,15 +77,14 @@ async function generateWriters( ) { console.log('Generating writers...') - let code = generateWriterCodeForTlEntries(apiSchema.entries, { - variableName: 'm', - }) + let code = generateWriterCodeForTlEntries( + [...apiSchema.entries, ...mtpSchema.entries], + { + variableName: 'm', + includeStaticSizes: true, + }, + ) - const mtpCode = generateWriterCodeForTlEntries(mtpSchema.entries, { - variableName: 'm', - includePrelude: false, - }) - code = code.substring(0, code.length - 1) + mtpCode.substring(7) code += '\nexports.default = m;' await writeFile(OUT_WRITERS_FILE, ESM_PRELUDE + code)