feat(tl): static object size computation

closes MTQ-21

packages/file-id/src/serialize-unique.ts

@@ -149,7 +149,7 @@ export function toUniqueFileId(
         }
         case 'web':
             writer = TlBinaryWriter.alloc(
-                {},
+                undefined,
                 Buffer.byteLength(inputLocation.url, 'utf-8') + 8,
             )
             writer.int(type)

packages/file-id/src/serialize.ts

@@ -21,15 +21,16 @@ export function toFileId(
     if (loc._ === 'web') type |= td.WEB_LOCATION_FLAG
     if (location.fileReference) type |= td.FILE_REFERENCE_FLAG
 
+    // overhead of the web file id:
+    // 8-16 bytes header,
+    // 8 bytes for access hash,
+    // up to 4 bytes for url
+    //
+    // longest file ids are around 80 bytes, so i guess
+    // we are safe with allocating 100 bytes
     const writer = TlBinaryWriter.alloc(
-        {},
+        undefined,
-        loc._ === 'web' ? // overhead of the web file id:
+        loc._ === 'web' ? Buffer.byteLength(loc.url, 'utf8') + 32 : 100,
-        // 8-16 bytes header,
-        // 8 bytes for access hash,
-        // up to 4 bytes for url
-            Buffer.byteLength(loc.url, 'utf8') + 32 : // longest file ids are around 80 bytes, so i guess
-        // we are safe with allocating 100 bytes
-            100,
     )
 
     writer.int(type)

packages/tl-runtime/src/writer.ts

@@ -10,6 +10,7 @@ const TWO_PWR_32_DBL = (1 << 16) * (1 << 16)
 export type TlWriterMap = Record<string, (w: any, val: any) => void> & {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     _bare?: Record<number, (w: any, val: any) => void>
+    _staticSize: Record<string, number>
 }
 
 /**
@@ -165,7 +166,10 @@ export class TlBinaryWriter {
      * @param objectMap  Writers map
      * @param size  Size of the writer's buffer
      */
-    static alloc(objectMap: TlWriterMap, size: number): TlBinaryWriter {
+    static alloc(
+        objectMap: TlWriterMap | undefined,
+        size: number,
+    ): TlBinaryWriter {
         return new TlBinaryWriter(objectMap, Buffer.allocUnsafe(size))
     }
 
@@ -202,7 +206,9 @@ export class TlBinaryWriter {
         knownSize = -1,
     ): Buffer {
         if (knownSize === -1) {
-            knownSize = TlSerializationCounter.countNeededBytes(objectMap, obj)
+            knownSize =
+                objectMap._staticSize[obj._] ||
+                TlSerializationCounter.countNeededBytes(objectMap, obj)
         }
 
         const writer = TlBinaryWriter.alloc(objectMap, knownSize)

packages/tl-runtime/tests/binary-writer.spec.ts

@@ -10,7 +10,7 @@ describe('TlBinaryWriter', () => {
     const testSingleMethod = (
         size: number,
         fn: (w: TlBinaryWriter) => void,
-        map: TlWriterMap = {},
+        map?: TlWriterMap,
     ): string => {
         const w = TlBinaryWriter.alloc(map, size)
         fn(w)
@@ -126,6 +126,8 @@ describe('TlBinaryWriter', () => {
             w.uint(0xbebf0c3d)
             w.vector(w.int, obj.vec)
         },
+        // eslint-disable-next-line
+        _staticSize: {} as any,
     }
 
     it('should write tg-encoded objects', () => {
@@ -218,6 +220,8 @@ describe('TlBinaryWriter', () => {
                     w.bytes(obj.pq)
                     w.vector(w.long, obj.serverPublicKeyFingerprints)
                 },
+                // eslint-disable-next-line
+                _staticSize: {} as any,
             }
 
             const length =

packages/tl-utils/src/calculator.ts

@@ -0,0 +1,96 @@
+import { TlEntry } from './types'
+
+const PRIMITIVES_SIZES: Record<string, number> = {
+    int: 4,
+    long: 8,
+    int53: 8,
+    int128: 16,
+    int256: 32,
+    double: 8,
+    boolFalse: 4,
+    boolTrue: 4,
+    bool: 4,
+    Bool: 4,
+    '#': 4,
+}
+
+export function calculateStaticSizes(
+    entries: TlEntry[],
+): Record<string, number> {
+    const staticSizes: Record<string, number> = {}
+    const unionSizes: Record<string, Record<string, number | null>> = {}
+    let changedInLastIteration = true
+
+    function getUnionStaticSize(name: string): number | null {
+        const values = Object.values(unionSizes[name] ?? {})
+        if (values.length === 0) return null
+
+        const first = values[0]
+        if (first === null) return null
+
+        for (const value of values) {
+            if (value !== first) return null
+        }
+
+        return first
+    }
+
+    function calculateStaticSize(entry: TlEntry): number | null {
+        if (entry.generics) return null // definitely not static sized
+
+        let size = 4 // constructor id
+
+        for (const arg of entry.arguments) {
+            if (arg.typeModifiers?.predicate) {
+                if (arg.type === 'true') {
+                    continue // zero-size type
+                }
+
+                // cant be static sized
+                return null
+            }
+
+            let unionSize
+
+            if (arg.type in PRIMITIVES_SIZES) {
+                size += PRIMITIVES_SIZES[arg.type]
+            } else if (arg.type in staticSizes) {
+                size += staticSizes[arg.type] - 4 // subtract constructor id
+            } else if ((unionSize = getUnionStaticSize(arg.type))) {
+                size += unionSize
+            } else {
+                // likely not static sized
+                return null
+            }
+        }
+
+        return size
+    }
+
+    while (changedInLastIteration) {
+        changedInLastIteration = false
+
+        for (const entry of entries) {
+            if (staticSizes[entry.name] !== undefined) continue
+
+            const size = calculateStaticSize(entry)
+
+            if (entry.kind === 'class') {
+                if (!unionSizes[entry.type]) {
+                    unionSizes[entry.type] = {}
+                }
+
+                unionSizes[entry.type][entry.name] = size
+            }
+
+            if (size === null) {
+                continue
+            }
+
+            staticSizes[entry.name] = size
+            changedInLastIteration = true
+        }
+    }
+
+    return staticSizes
+}

packages/tl-utils/src/codegen/writer.ts

@@ -1,3 +1,4 @@
+import { calculateStaticSizes } from '../calculator'
 import { computeConstructorIdFromEntry } from '../ctor-id'
 import { TL_PRIMITIVES, TlEntry } from '../types'
 import { snakeToCamel } from './utils'
@@ -20,6 +21,11 @@ export interface WriterCodegenOptions {
      */
     includePrelude?: boolean
 
+    /**
+     * Whether to include `_staticSize` field
+     */
+    includeStaticSizes?: boolean
+
     /**
      * Whether to generate bare writer (without constructor id write)
      */
@@ -31,6 +37,7 @@ const DEFAULT_OPTIONS: WriterCodegenOptions = {
     variableName: 'm',
     includePrelude: true,
     bare: false,
+    includeStaticSizes: false,
 }
 
 const TL_WRITER_PRELUDE =
@@ -168,7 +175,10 @@ export function generateWriterCodeForTlEntries(
     entries: TlEntry[],
     params = DEFAULT_OPTIONS,
 ): string {
-    const { includePrelude, variableName } = { ...DEFAULT_OPTIONS, ...params }
+    const { includePrelude, variableName, includeStaticSizes } = {
+        ...DEFAULT_OPTIONS,
+        ...params,
+    }
 
     let ret = ''
     if (includePrelude) ret += TL_WRITER_PRELUDE
@@ -201,7 +211,19 @@ export function generateWriterCodeForTlEntries(
                     bare: true,
                 }) + '\n'
         })
-        ret += '}'
+        ret += '},\n'
+    }
+
+    if (includeStaticSizes) {
+        ret += '_staticSize:{\n'
+
+        const staticSizes = calculateStaticSizes(entries)
+
+        Object.keys(staticSizes).forEach((name) => {
+            ret += `'${name}':${staticSizes[name]},\n`
+        })
+
+        ret += '},\n'
     }
 
     return ret + '}'

packages/tl-utils/src/merge.ts

@@ -1,5 +1,5 @@
 import { computeConstructorIdFromEntry } from './ctor-id'
-import { TlEntry, TlFullSchema } from './types'
+import { TlArgument, TlEntry, TlFullSchema } from './types'
 
 /**
  * Merge multiple TL entries into a single entry.
@@ -18,27 +18,28 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string {
         typeModifiers: first.typeModifiers,
         id: first.id,
         comment: first.comment,
-        generics: first.generics,
+        generics: first.generics?.map((it) => ({ ...it })),
-        arguments: first.arguments,
+        arguments: first.arguments.map((it) => ({ ...it })),
     }
 
     if (result.id === 0) {
         result.id = computeConstructorIdFromEntry(result)
     }
 
-    const argsIndex: Record<string, true> = {}
+    const argsIndex: Record<string, TlArgument> = {}
     const flagsLastIndex: Record<string, number> = {}
 
     result.arguments.forEach((arg, idx) => {
-        argsIndex[arg.name] = true
+        argsIndex[arg.name] = arg
 
         if (arg.type === '#') {
             flagsLastIndex[arg.name] = idx
         }
-        // if (arg.predicate) {
+
-        //     const flagsField = arg.predicate.split('.')[0]
+        if (arg.typeModifiers?.predicate) {
-        //     flagsLastIndex[flagsField] = idx
+            const flagsField = arg.typeModifiers.predicate.split('.')[0]
-        // }
+            flagsLastIndex[flagsField] = idx
+        }
     })
 
     for (let i = 1; i < entries.length; i++) {
@@ -89,7 +90,7 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string {
                 // targetIdx *must* exist, otherwise ids wouldn't match
 
                 result.arguments.splice(targetIdx + 1, 0, entryArgument)
-                argsIndex[entryArgument.name] = true
+                argsIndex[entryArgument.name] = entryArgument
 
                 // update last indexes
                 // we also need to update subsequent flags if there are any
@@ -98,10 +99,17 @@ export function mergeTlEntries(entries: TlEntry[]): TlEntry | string {
                         flagsLastIndex[flag]++
                     }
                 })
+
+                continue
             }
 
             // args exists both in result and current entry
             // since ctor ids match, it must be the same, so we don't need to check
+            // we still need to merge comments though
+
+            if (!resultArgument.comment && entryArgument.comment) {
+                resultArgument.comment = entryArgument.comment
+            }
         }
     }
 

packages/tl-utils/src/types.ts

@@ -426,5 +426,5 @@ export const TL_PRIMITIVES = {
     Bool: 1,
     true: 1,
     null: 1,
-    Object: true,
+    Object: 1,
 } as const

packages/tl-utils/tests/calculator.spec.ts

@@ -0,0 +1,67 @@
+import { expect } from 'chai'
+import { describe, it } from 'mocha'
+
+import { parseTlToEntries } from '../src'
+import { calculateStaticSizes } from '../src/calculator'
+
+describe('calculateStaticSizes', () => {
+    const test = (tl: string, expected: object) => {
+        expect(calculateStaticSizes(parseTlToEntries(tl))).eql(expected)
+    }
+
+    it('computes for constructors without parameters', () => {
+        test('auth.logOut = Bool;', { 'auth.logOut': 4 })
+    })
+
+    it('computes for constructors with static parameters', () => {
+        test(
+            'auth.exportAuthorization#e5bfffcd dc_id:int = auth.ExportedAuthorization;',
+            { 'auth.exportAuthorization': 8 },
+        )
+    })
+
+    it('correctly skips true fields', () => {
+        test(
+            'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int = help.PromoData;',
+            { 'help.promoData': 12 },
+        )
+    })
+
+    it('correctly skips constructors with predicated fields', () => {
+        test(
+            'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:Peer psa_type:flags.1?string psa_message:flags.2?string = help.PromoData;',
+            {},
+        )
+    })
+
+    it('correctly skips constructors with non-static fields', () => {
+        test(
+            'help.promoData#8c39793f psa_type:string psa_message:string = help.PromoData;',
+            {},
+        )
+    })
+
+    it('correctly handles static-sized children', () => {
+        test(
+            'peerUser#9db1bc6d user_id:int53 = Peer;\n' +
+                'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:peerUser = help.PromoData;',
+            {
+                peerUser: 12,
+                'help.promoData': 20,
+            },
+        )
+    })
+
+    it('correctly handles static-sized union children', () => {
+        test(
+            'peerUser#9db1bc6d user_id:int53 = Peer;\n' +
+                'peerChannel#9db1bc6d channel_id:int53 = Peer;\n' +
+                'help.promoData#8c39793f flags:# proxy:flags.0?true expires:int peer:Peer = help.PromoData;',
+            {
+                peerUser: 12,
+                peerChannel: 12,
+                'help.promoData': 24,
+            },
+        )
+    })
+})

packages/tl-utils/tests/codegen/writer.spec.ts

@@ -137,7 +137,7 @@ describe('generateWriterCodeForTlEntry', () => {
                 'future_salt':function(w,v){w.uint(155834844);w.bytes(h(v,'salt'));},
                 'future_salts':function(w,v){w.uint(2924480661);w.vector(m._bare[155834844],h(v,'salts'),1);m._bare[155834844](w,h(v,'current'));},
                 _bare:{
-                    155834844:function(w,v){w.bytes(h(v,'salt'));},
+                    155834844:function(w=this,v){w.bytes(h(v,'salt'));},
             }}`.replace(/^\s+/gm, ''),
         )
     })

packages/tl-utils/tests/merge.spec.ts

@@ -1,10 +1,14 @@
 import { expect } from 'chai'
 import { describe, it } from 'mocha'
 
-import { mergeTlEntries, mergeTlSchemas } from '../src/merge'
+import {
-import { parseTlToEntries } from '../src/parse'
+    mergeTlEntries,
-import { parseFullTlSchema, writeTlEntriesToString } from '../src/schema'
+    mergeTlSchemas,
-import { writeTlEntryToString } from '../src/stringify'
+    parseFullTlSchema,
+    parseTlToEntries,
+    writeTlEntriesToString,
+    writeTlEntryToString,
+} from '../src'
 
 describe('mergeTlEntries', () => {
     const test = (tl: string, expected: string) => {
@@ -99,10 +103,10 @@ describe('mergeTlSchemas', () => {
                 ['---functions---', 'testMethod = Test;'],
             ],
             0,
-            'testClass = Test;',
+            'testClass#5d60a438 = Test;',
-            'testClass2 = Test;',
+            'testClass2#39c5c841 = Test;',
             '---functions---',
-            'testMethod = Test;',
+            'testMethod#87d8a7d2 = Test;',
         )
     })
 
@@ -114,7 +118,7 @@ describe('mergeTlSchemas', () => {
                 ['test foo:flags.0?true bar:flags.0?true = Test;'],
             ],
             0,
-            'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;',
+            'test#1c173316 bar:flags.0?true foo:flags.0?true = Test;',
         )
     })
 
@@ -126,7 +130,7 @@ describe('mergeTlSchemas', () => {
                 ['test baz:int = Test;'],
             ],
             0,
-            'test foo:int = Test;',
+            'test#4f6455cd foo:int = Test;',
         )
         await test(
             [
@@ -135,7 +139,7 @@ describe('mergeTlSchemas', () => {
                 ['test baz:int = Test;'],
             ],
             1,
-            'test foo:int = Test;',
+            'test#3e993a74 bar:int = Test;',
         )
         await test(
             [['test foo:int = Test;'], [], ['test bar:int = Test;']],
@@ -156,7 +160,7 @@ describe('mergeTlSchemas', () => {
             ],
             0,
             '// @description test ctor',
-            'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;',
+            'test#1c173316 bar:flags.0?true foo:flags.0?true = Test;',
         )
     })
 
@@ -164,13 +168,17 @@ describe('mergeTlSchemas', () => {
         await test(
             [
                 ['test foo:flags.0?true = Test;'],
-                ['// @bar bar comment', 'test bar:flags.0?true = Test;'],
                 [
-                    '// @foo foo comment',
+                    '// @description test @bar bar comment',
+                    'test bar:flags.0?true = Test;',
+                ],
+                [
+                    '// @description test @foo foo comment',
                     'test foo:flags.0?true bar:flags.0?true = Test;',
                 ],
             ],
             0,
+            '// @description test',
             '// @foo foo comment',
             '// @bar bar comment',
             'test#1c173316 foo:flags.0?true bar:flags.0?true = Test;',

packages/tl/binary/writer.d.ts

@@ -1,5 +1,6 @@
 declare const __tlWriterMap: Record<string, (w: unknown, val: unknown) => void> & {
     _bare: Record<number, (r: unknown) => unknown>
+    _staticSize: Record<string, number>
 }
 // eslint-disable-next-line import/no-default-export
 export default __tlWriterMap

packages/tl/scripts/gen-code.ts

@@ -77,15 +77,14 @@ async function generateWriters(
 ) {
     console.log('Generating writers...')
 
-    let code = generateWriterCodeForTlEntries(apiSchema.entries, {
+    let code = generateWriterCodeForTlEntries(
-        variableName: 'm',
+        [...apiSchema.entries, ...mtpSchema.entries],
-    })
+        {
+            variableName: 'm',
+            includeStaticSizes: true,
+        },
+    )
 
-    const mtpCode = generateWriterCodeForTlEntries(mtpSchema.entries, {
-        variableName: 'm',
-        includePrelude: false,
-    })
-    code = code.substring(0, code.length - 1) + mtpCode.substring(7)
     code += '\nexports.default = m;'
 
     await writeFile(OUT_WRITERS_FILE, ESM_PRELUDE + code)