teidesu/mtcute
1
0
Fork 0
Code Issues 15 Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat(tl): static object size computation

Browse source 
closes MTQ-21
This commit is contained in:
alina 🌸 2023-09-20 18:37:26 +03:00
parent 5a3b101c9f
commit b8f63b0634
Signed by: teidesu
SSH key fingerprint: SHA256:uNeCpw6aTSU4aIObXLvHfLkDa82HWH9EiOj9AXOIRpI
13 changed files with 259 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/file-id/src/serialize-unique.ts
View file

@ -149,7 +149,7 @@ export function toUniqueFileId(
}
case 'web':
writer = TlBinaryWriter.alloc(
{},
undefined,
Buffer.byteLength(inputLocation.url, 'utf-8') + 8,
)
writer.int(type)

11
packages/file-id/src/serialize.ts
View file

@ -21,15 +21,16 @@ export function toFileId(
if (loc._ === 'web') type |= td.WEB_LOCATION_FLAG
if (location.fileReference) type |= td.FILE_REFERENCE_FLAG
const writer = TlBinaryWriter.alloc(
{},
loc._ === 'web' ? // overhead of the web file id:
// overhead of the web file id:
// 8-16 bytes header,
// 8 bytes for access hash,
// up to 4 bytes for url
Buffer.byteLength(loc.url, 'utf8') + 32 : // longest file ids are around 80 bytes, so i guess
//
// longest file ids are around 80 bytes, so i guess
// we are safe with allocating 100 bytes
100,
const writer = TlBinaryWriter.alloc(
undefined,
loc._ === 'web' ? Buffer.byteLength(loc.url, 'utf8') + 32 : 100,
)
writer.int(type)

10
packages/tl-runtime/src/writer.ts
View file

@ -10,6 +10,7 @@ const TWO_PWR_32_DBL = (1 << 16) * (1 << 16)
export type TlWriterMap = Record<string, (w: any, val: any) => void> & {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
_bare?: Record<number, (w: any, val: any) => void>
_staticSize: Record<string, number>
}
/**
@ -165,7 +166,10 @@ export class TlBinaryWriter {
* @param objectMap Writers map
* @param size Size of the writer's buffer
*/
static alloc(objectMap: TlWriterMap, size: number): TlBinaryWriter {
static alloc(
objectMap: TlWriterMap | undefined,
size: number,
): TlBinaryWriter {
return new TlBinaryWriter(objectMap, Buffer.allocUnsafe(size))
}
@ -202,7 +206,9 @@ export class TlBinaryWriter {
knownSize = -1,
): Buffer {
if (knownSize === -1) {
knownSize = TlSerializationCounter.countNeededBytes(objectMap, obj)
knownSize =
objectMap._staticSize[obj._] ||
TlSerializationCounter.countNeededBytes(objectMap, obj)
}
const writer = TlBinaryWriter.alloc(objectMap, knownSize)

6
packages/tl-runtime/tests/binary-writer.spec.ts
View file

@ -10,7 +10,7 @@ describe('TlBinaryWriter', () => {
const testSingleMethod = (
size: number,
fn: (w: TlBinaryWriter) => void,
map: TlWriterMap = {},
map?: TlWriterMap,
): string => {
const w = TlBinaryWriter.alloc(map, size)
fn(w)
@ -126,6 +126,8 @@ describe('TlBinaryWriter', () => {
w.uint(0xbebf0c3d)
w.vector(w.int, obj.vec)
},
// eslint-disable-next-line
_staticSize: {} as any,
}
it('should write tg-encoded objects', () => {
@ -218,6 +220,8 @@ describe('TlBinaryWriter', () => {
w.bytes(obj.pq)
w.vector(w.long, obj.serverPublicKeyFingerprints)
},
// eslint-disable-next-line
_staticSize: {} as any,
}
const length =

96
packages/tl-utils/src/calculator.ts Normal file
View file

@ -0,0 +1,96 @@
import { TlEntry } from './types'
const PRIMITIVES_SIZES: Record<string, number> = {
int: 4,
long: 8,
int53: 8,
int128: 16,
int256: 32,
double: 8,
boolFalse: 4,
boolTrue: 4,
bool: 4,
Bool: 4,
'#': 4,
}
export function calculateStaticSizes(
entries: TlEntry[],
): Record<string, number> {
const staticSizes: Record<string, number> = {}
const unionSizes: Record<string, Record<string, number | null>> = {}
let changedInLastIteration = true
function getUnionStaticSize(name: string): number | null {
const values = Object.values(unionSizes[name] ?? {})
if (values.length === 0) return null
const first = values[0]
if (first === null) return null
for (const value of values) {
if (value !== first) return null
}
return first
}
function calculateStaticSize(entry: TlEntry): number | null {
if (entry.generics) return null // definitely not static sized
let size = 4 // constructor id
for (const arg of entry.arguments) {
if (arg.typeModifiers?.predicate) {
if (arg.type === 'true') {
continue // zero-size type
}
// cant be static sized
return null
}
let unionSize
if (arg.type in PRIMITIVES_SIZES) {
size += PRIMITIVES_SIZES[arg.type]
} else if (arg.type in staticSizes) {
size += staticSizes[arg.type] - 4 // subtract constructor id
} else if ((unionSize = getUnionStaticSize(arg.type))) {
size += unionSize
} else {
// likely not static sized
return null
}
}
return size
}
while (changedInLastIteration) {
changedInLastIteration = false
for (const entry of entries) {
if (staticSizes[entry.name] !== undefined) continue
const size = calculateStaticSize(entry)
if (entry.kind === 'class') {
if (!unionSizes[entry.type]) {
unionSizes[entry.type] = {}
}
unionSizes[entry.type][entry.name] = size
}
if (size === null) {
continue
}
staticSizes[entry.name] = size
changedInLastIteration = true
}
}
return staticSizes
}

26
packages/tl-utils/src/codegen/writer.ts
View file

@ -1,3 +1,4 @@
import { calculateStaticSizes } from '../calculator'
import { computeConstructorIdFromEntry } from '../ctor-id'
import { TL_PRIMITIVES, TlEntry } from '../types'
import { snakeToCamel } from './utils'
@ -20,6 +21,11 @@ export interface WriterCodegenOptions {
*/
includePrelude?: boolean
/**
* Whether to include `_staticSize` field
*/
includeStaticSizes?: boolean
/**
* Whether to generate bare writer (without constructor id write)
*/
@ -31,6 +37,7 @@ const DEFAULT_OPTIONS: WriterCodegenOptions = {
variableName: 'm',
includePrelude: true,
bare: false,
includeStaticSizes: false,
}
const TL_WRITER_PRELUDE =
@ -168,7 +175,10 @@ export function generateWriterCodeForTlEntries(
entries: TlEntry[],
params = DEFAULT_OPTIONS,
): string {
const { includePrelude, variableName } = { ...DEFAULT_OPTIONS, ...params }
const { includePrelude, variableName, includeStaticSizes } = {
...DEFAULT_OPTIONS,
...params,
}
let ret = ''
if (includePrelude) ret += TL_WRITER_PRELUDE
@ -201,7 +211,19 @@ export function generateWriterCodeForTlEntries(
bare: true,
}) + '\n'
})
ret += '}'
ret += '},\n'
}
if (includeStaticSizes) {
ret += '_staticSize:{\n'
const staticSizes = calculateStaticSizes(entries)
Object.keys(staticSizes).forEach((name) => {
ret += `'${name}':${staticSizes[name]},\n`
})
ret += '},\n'
}
return ret + '}'

28
packages/tl-utils/src/merge.ts
View file

@ -1,5 +1,5 @@
import { computeConstructorIdFromEntry } from './ctor-id'
import { TlEntry, TlFullSchema } from './types'
import { TlArgument, TlEntry, TlFullSchema } from './types'
/**
* Merge multiple TL entries into a single entry.
@ -18,27 +18,28 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string {
typeModifiers: first.typeModifiers,
id: first.id,
comment: first.comment,
generics: first.generics,
arguments: first.arguments,
generics: first.generics?.map((it) => ({ ...it })),
arguments: first.arguments.map((it) => ({ ...it })),
}
if (result.id === 0) {
result.id = computeConstructorIdFromEntry(result)
}
const argsIndex: Record<string, true> = {}
const argsIndex: Record<string, TlArgument> = {}
const flagsLastIndex: Record<string, number> = {}
result.arguments.forEach((arg, idx) => {
argsIndex[arg.name] = true
argsIndex[arg.name] = arg
if (arg.type === '#') {
flagsLastIndex[arg.name] = idx
}
// if (arg.predicate) {
// const flagsField = arg.predicate.split('.')[0]
// flagsLastIndex[flagsField] = idx
// }
if (arg.typeModifiers?.predicate) {
const flagsField = arg.typeModifiers.predicate.split('.')[0]
flagsLastIndex[flagsField] = idx
}
})
for (let i = 1; i < entries.length; i++) {
@ -89,7 +90,7 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string {
// targetIdx *must* exist, otherwise ids wouldn't match
result.arguments.splice(targetIdx + 1, 0, entryArgument)
argsIndex[entryArgument.name] = true
argsIndex[entryArgument.name] = entryArgument
// update last indexes
// we also need to update subsequent flags if there are any
@ -98,10 +99,17 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string {
flagsLastIndex[flag]++
}
})
continue
}
// args exists both in result and current entry
// since ctor ids match, it must be the same, so we don't need to check
// we still need to merge comments though
if (!resultArgument.comment && entryArgument.comment) {
resultArgument.comment = entryArgument.comment
}
}
}

2
packages/tl-utils/src/types.ts
View file

@ -426,5 +426,5 @@ export const TL_PRIMITIVES = {
Bool: 1,
true: 1,
null: 1,
Object: true,
Object: 1,
} as const

67
packages/tl-utils/tests/calculator.spec.ts Normal file
View file

@ -0,0 +1,67 @@
import { expect } from 'chai'
import { describe, it } from 'mocha'
import { parseTlToEntries } from '../src'
import { calculateStaticSizes } from '../src/calculator'
describe('calculateStaticSizes', () => {
const test = (tl: string, expected: object) => {
expect(calculateStaticSizes(parseTlToEntries(tl))).eql(expected)
}
it('computes for constructors without parameters', () => {
test('auth.logOut = Bool;', { 'auth.logOut': 4 })
})
it('computes for constructors with static parameters', () => {
test(
'auth.exportAuthorization#e5bfffcd dc_id:int = auth.ExportedAuthorization;',
{ 'auth.exportAuthorization': 8 },
)
})
it('correctly skips true fields', () => {
test(
'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int = help.PromoData;',
{ 'help.promoData': 12 },
)
})
it('correctly skips constructors with predicated fields', () => {
test(
'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:Peer psa_type:flags.1?string psa_message:flags.2?string = help.PromoData;',
{},
)
})
it('correctly skips constructors with non-static fields', () => {
test(
'help.promoData#8c39793f psa_type:string psa_message:string = help.PromoData;',
{},
)
})
it('correctly handles static-sized children', () => {
test(
'peerUser#9db1bc6d user_id:int53 = Peer;\n' +
'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:peerUser = help.PromoData;',
{
peerUser: 12,
'help.promoData': 20,
},
)
})
it('correctly handles static-sized union children', () => {
test(
'peerUser#9db1bc6d user_id:int53 = Peer;\n' +
'peerChannel#9db1bc6d channel_id:int53 = Peer;\n' +
'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:Peer = help.PromoData;',
{
peerUser: 12,
peerChannel: 12,
'help.promoData': 24,
},
)
})
})

2
packages/tl-utils/tests/codegen/writer.spec.ts
View file

@ -137,7 +137,7 @@ describe('generateWriterCodeForTlEntry', () => {
'future_salt':function(w,v){w.uint(155834844);w.bytes(h(v,'salt'));},
'future_salts':function(w,v){w.uint(2924480661);w.vector(m._bare[155834844],h(v,'salts'),1);m._bare[155834844](w,h(v,'current'));},
_bare:{
155834844:function(w,v){w.bytes(h(v,'salt'));},
155834844:function(w=this,v){w.bytes(h(v,'salt'));},
}}`.replace(/^\s+/gm, ''),
)
})

34
packages/tl-utils/tests/merge.spec.ts
View file

@ -1,10 +1,14 @@
import { expect } from 'chai'
import { describe, it } from 'mocha'
import { mergeTlEntries, mergeTlSchemas } from '../src/merge'
import { parseTlToEntries } from '../src/parse'
import { parseFullTlSchema, writeTlEntriesToString } from '../src/schema'
import { writeTlEntryToString } from '../src/stringify'
import {
mergeTlEntries,
mergeTlSchemas,
parseFullTlSchema,
parseTlToEntries,
writeTlEntriesToString,
writeTlEntryToString,
} from '../src'
describe('mergeTlEntries', () => {
const test = (tl: string, expected: string) => {
@ -99,10 +103,10 @@ describe('mergeTlSchemas', () => {
['---functions---', 'testMethod = Test;'],
],
0,
'testClass = Test;',
'testClass2 = Test;',
'testClass#5d60a438 = Test;',
'testClass2#39c5c841 = Test;',
'---functions---',
'testMethod = Test;',
'testMethod#87d8a7d2 = Test;',
)
})
@ -114,7 +118,7 @@ describe('mergeTlSchemas', () => {
['test foo:flags.0?true bar:flags.0?true = Test;'],
],
0,
'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;',
'test#1c173316 bar:flags.0?true foo:flags.0?true = Test;',
)
})
@ -126,7 +130,7 @@ describe('mergeTlSchemas', () => {
['test baz:int = Test;'],
],
0,
'test foo:int = Test;',
'test#4f6455cd foo:int = Test;',
)
await test(
[
@ -135,7 +139,7 @@ describe('mergeTlSchemas', () => {
['test baz:int = Test;'],
],
1,
'test foo:int = Test;',
'test#3e993a74 bar:int = Test;',
)
await test(
[['test foo:int = Test;'], [], ['test bar:int = Test;']],
@ -156,7 +160,7 @@ describe('mergeTlSchemas', () => {
],
0,
'// @description test ctor',
'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;',
'test#1c173316 bar:flags.0?true foo:flags.0?true = Test;',
)
})
@ -164,13 +168,17 @@ describe('mergeTlSchemas', () => {
await test(
[
['test foo:flags.0?true = Test;'],
['// @bar bar comment', 'test bar:flags.0?true = Test;'],
[
'// @foo foo comment',
'// @description test @bar bar comment',
'test bar:flags.0?true = Test;',
],
[
'// @description test @foo foo comment',
'test foo:flags.0?true bar:flags.0?true = Test;',
],
],
0,
'// @description test',
'// @foo foo comment',
'// @bar bar comment',
'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;',

1
packages/tl/binary/writer.d.ts vendored
View file

@ -1,5 +1,6 @@
declare const __tlWriterMap: Record<string, (w: unknown, val: unknown) => void> & {
_bare: Record<number, (r: unknown) => unknown>
_staticSize: Record<string, number>
}
// eslint-disable-next-line import/no-default-export
export default __tlWriterMap

13
packages/tl/scripts/gen-code.ts
View file

@ -77,15 +77,14 @@ async function generateWriters(
) {
console.log('Generating writers...')
let code = generateWriterCodeForTlEntries(apiSchema.entries, {
let code = generateWriterCodeForTlEntries(
[...apiSchema.entries, ...mtpSchema.entries],
{
variableName: 'm',
})
includeStaticSizes: true,
},
)
const mtpCode = generateWriterCodeForTlEntries(mtpSchema.entries, {
variableName: 'm',
includePrelude: false,
})
code = code.substring(0, code.length - 1) + mtpCode.substring(7)
code += '\nexports.default = m;'
await writeFile(OUT_WRITERS_FILE, ESM_PRELUDE + code)