diff --git a/packages/tl-utils/package.json b/packages/tl-utils/package.json index d6216065..7bf3c428 100644 --- a/packages/tl-utils/package.json +++ b/packages/tl-utils/package.json @@ -12,6 +12,22 @@ "docs": "typedoc", "build": "pnpm run -w build-package tl-utils" }, + "exports": { + ".": "./src/index.ts", + "./json.js": "./src/json/index.ts" + }, + "distOnlyFields": { + "exports": { + ".": { + "import": "./dist/esm/index.js", + "require": "./dist/cjs/index.js" + }, + "./json.js": { + "import": "./dist/esm/json/index.js", + "require": "./dist/cjs/json/index.js" + } + } + }, "dependencies": { "crc-32": "1.2.0" }, diff --git a/packages/tl-utils/src/ctor-id.ts b/packages/tl-utils/src/ctor-id.ts index 0c773fba..5d0931d9 100644 --- a/packages/tl-utils/src/ctor-id.ts +++ b/packages/tl-utils/src/ctor-id.ts @@ -9,5 +9,7 @@ import { TlEntry } from './types.js' * @param entry TL entry */ export function computeConstructorIdFromEntry(entry: TlEntry): number { - return CRC32.str(writeTlEntryToString(entry, true)) >>> 0 + const str = writeTlEntryToString(entry, true) + + return CRC32.str(str) >>> 0 } diff --git a/packages/tl-utils/src/json/from-json.test.ts b/packages/tl-utils/src/json/from-json.test.ts new file mode 100644 index 00000000..70e23323 --- /dev/null +++ b/packages/tl-utils/src/json/from-json.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, it } from 'vitest' + +import { computeConstructorIdFromEntry } from '../ctor-id.js' +import { TlEntry } from '../types.js' +import { parseTlEntriesFromJson } from './from-json.js' + +describe('parseTlEntriesFromJson', () => { + const test = (json: object, expected: TlEntry[], params?: Parameters[1]) => { + const entries = parseTlEntriesFromJson(json, params) + expect(entries).toEqual(expected) + + for (const entry of entries) { + expect(entry.id).to.equal(computeConstructorIdFromEntry(entry), `ID for ${entry.name}`) + } + } + + it('parses simple constructors', () => { + test( + { + constructors: [ + { + id: '-1132882121', + predicate: 'boolFalse', + params: [], + type: 'Bool', + }, + { + id: '-1720552011', + predicate: 'boolTrue', + params: [], + type: 'Bool', + }, + ], + methods: [], + }, + [ + { + arguments: [], + id: 3162085175, + kind: 'class', + name: 'boolFalse', + type: 'Bool', + }, + { + arguments: [], + id: 2574415285, + kind: 'class', + name: 'boolTrue', + type: 'Bool', + }, + ], + { keepPrimitives: true }, + ) + }) + + it('parses simple arguments', () => { + test( + { + constructors: [ + { + id: '-122978821', + predicate: 'inputMediaContact', + params: [ + { name: 'phone_number', type: 'string' }, + { name: 'first_name', type: 'string' }, + { name: 'last_name', type: 'string' }, + { name: 'vcard', type: 'string' }, + ], + type: 'InputMedia', + }, + ], + methods: [], + }, + [ + { + arguments: [ + { name: 'phone_number', type: 'string' }, + { name: 'first_name', type: 'string' }, + { name: 'last_name', type: 'string' }, + { name: 'vcard', type: 'string' }, + ], + id: 4171988475, + kind: 'class', + name: 'inputMediaContact', + type: 'InputMedia', + }, + ], + ) + }) + + it('parses predicated arguments', () => { + test( + { + constructors: [ + { + id: '-1110593856', + predicate: 'inputChatUploadedPhoto', + params: [ + { name: 'flags', type: '#' }, + { name: 'file', type: 'flags.0?InputFile' }, + { name: 'video', type: 'flags.1?InputFile' }, + { name: 'video_start_ts', type: 'flags.2?double' }, + { name: 'video_emoji_markup', type: 'flags.3?VideoSize' }, + ], + type: 'InputChatPhoto', + }, + ], + methods: [], + }, + [ + { + arguments: [ + { + name: 'flags', + type: '#', + typeModifiers: undefined, + }, + { + name: 'file', + type: 'InputFile', + typeModifiers: { + predicate: 'flags.0', + }, + }, + { + name: 'video', + type: 'InputFile', + typeModifiers: { + predicate: 'flags.1', + }, + }, + { + name: 'video_start_ts', + type: 'double', + typeModifiers: { + predicate: 'flags.2', + }, + }, + { + name: 'video_emoji_markup', + type: 'VideoSize', + typeModifiers: { + predicate: 'flags.3', + }, + }, + ], + id: 3184373440, + kind: 'class', + name: 'inputChatUploadedPhoto', + type: 'InputChatPhoto', + }, + ], + ) + }) + + it('parses vector arguments', () => { + test( + { + constructors: [], + methods: [ + { + id: '1779249670', + method: 'account.unregisterDevice', + params: [ + { name: 'token_type', type: 'int' }, + { name: 'token', type: 'string' }, + { name: 'other_uids', type: 'Vector' }, + ], + type: 'Bool', + }, + { + id: '227648840', + method: 'users.getUsers', + params: [ + { + name: 'id', + type: 'Vector', + }, + ], + type: 'Vector', + }, + ], + }, + [ + { + arguments: [ + { + name: 'token_type', + type: 'int', + typeModifiers: undefined, + }, + { + name: 'token', + type: 'string', + typeModifiers: undefined, + }, + { + name: 'other_uids', + type: 'long', + typeModifiers: { + isVector: true, + }, + }, + ], + id: 1779249670, + kind: 'method', + name: 'account.unregisterDevice', + type: 'Bool', + }, + { + arguments: [ + { + name: 'id', + type: 'InputUser', + typeModifiers: { + isVector: true, + }, + }, + ], + id: 227648840, + kind: 'method', + name: 'users.getUsers', + type: 'User', + typeModifiers: { + isVector: true, + }, + }, + ], + { parseMethodTypes: true }, + ) + }) +}) diff --git a/packages/tl-utils/src/json/from-json.ts b/packages/tl-utils/src/json/from-json.ts new file mode 100644 index 00000000..72281428 --- /dev/null +++ b/packages/tl-utils/src/json/from-json.ts @@ -0,0 +1,99 @@ +import { TL_PRIMITIVES, TlArgument, TlEntry } from '../types.js' +import { parseArgumentType } from '../utils.js' +import { parseTlSchemaFromJson, TlParamJson } from './types.js' + +function paramsToArguments(params: TlParamJson[]): TlArgument[] { + return params.map((p) => { + const [type, modifiers] = parseArgumentType(p.type) + + return { + name: p.name, + type, + typeModifiers: Object.keys(modifiers).length ? modifiers : undefined, + } + }) +} + +export function parseTlEntriesFromJson( + json: object, + params?: { + /** + * Prefix to be applied to all types + */ + prefix?: string + + /** + * Whether to parse typeModifiers for method return types + */ + parseMethodTypes?: boolean + + /** + * Whether to keep primitives + */ + keepPrimitives?: boolean + }, +): TlEntry[] { + const { parseMethodTypes, keepPrimitives, prefix = '' } = params ?? {} + const schema = parseTlSchemaFromJson(json) + + const ret: TlEntry[] = [] + const entries: Record = {} + const unions: Record = {} + + schema.constructors.forEach((c) => { + if (!keepPrimitives && (c.predicate in TL_PRIMITIVES || c.type in TL_PRIMITIVES)) return + + const entry: TlEntry = { + id: Number(c.id) >>> 0, + kind: 'class', + name: prefix + c.predicate, + type: c.type, + arguments: paramsToArguments(c.params), + } + + entries[entry.name] = entry + ret.push(entry) + + if (c.type in unions) { + unions[c.type].push(entry) + } else { + unions[c.type] = [entry] + } + }) + + schema.methods.forEach((m) => { + const entry: TlEntry = { + id: Number(m.id) >>> 0, + kind: 'method', + name: prefix + m.method, + type: m.type, + arguments: paramsToArguments(m.params), + } + + if (parseMethodTypes) { + const [type, modifiers] = parseArgumentType(entry.type) + entry.type = type + + if (Object.keys(modifiers).length) { + entry.typeModifiers = modifiers + } + + // since constructors were all already processed, we can put return type ctor id here + if (type in unions && unions[type].length === 1) { + if (!entry.typeModifiers) entry.typeModifiers = {} + + entry.typeModifiers.constructorId = unions[type][0].id + } else if (type in entries) { + if (!entry.typeModifiers) entry.typeModifiers = {} + + entry.typeModifiers.isBareType = true + entry.typeModifiers.constructorId = entries[type].id + } + } + + entries[entry.name] = entry + ret.push(entry) + }) + + return ret +} diff --git a/packages/tl-utils/src/json/index.ts b/packages/tl-utils/src/json/index.ts new file mode 100644 index 00000000..83fb50db --- /dev/null +++ b/packages/tl-utils/src/json/index.ts @@ -0,0 +1,2 @@ +export * from './from-json.js' +export * from './types.js' diff --git a/packages/tl-utils/src/json/types.ts b/packages/tl-utils/src/json/types.ts new file mode 100644 index 00000000..e1bb0da0 --- /dev/null +++ b/packages/tl-utils/src/json/types.ts @@ -0,0 +1,107 @@ +function assertObject(obj: object): asserts obj is object { + if (typeof obj !== 'object' || obj === null) { + throw new Error('Expected object') + } +} + +interface TypeofToType { + string: string + number: number + boolean: boolean + object: object +} + +function assertFieldType( + obj: object, + field: Field, + type: Type, +): asserts obj is { [K in Field]: TypeofToType[Type] } { + // eslint-disable-next-line + const typeof_ = typeof (obj as any)[field] + + if (typeof_ !== type) { + throw new Error(`Expected field ${field} to be of type ${type} (got ${typeof_})`) + } +} + +export interface TlParamJson { + name: string + type: string +} + +export function parseTlParamFromJson(obj: object): TlParamJson { + assertObject(obj) + assertFieldType(obj, 'name', 'string') + assertFieldType(obj, 'type', 'string') + + return obj +} + +function assertFieldParams(obj: object): asserts obj is { params: TlParamJson[] } { + assertFieldType(obj, 'params', 'object') + + if (!Array.isArray(obj.params)) { + throw new Error('Expected field params to be an array') + } + + obj.params.forEach(parseTlParamFromJson) // will throw if invalid +} + +export interface TlConstructorJson { + id: string + type: string + predicate: string + params: TlParamJson[] +} + +export function parseTlConstructorFromJson(obj: object): TlConstructorJson { + assertObject(obj) + assertFieldType(obj, 'id', 'string') + assertFieldType(obj, 'type', 'string') + assertFieldType(obj, 'predicate', 'string') + assertFieldParams(obj) + + return obj +} + +export interface TlMethodJson { + id: string + type: string + method: string + params: TlParamJson[] +} + +export function parseTlMethodFromJson(obj: object): TlMethodJson { + assertObject(obj) + assertFieldType(obj, 'id', 'string') + assertFieldType(obj, 'type', 'string') + assertFieldType(obj, 'method', 'string') + assertFieldParams(obj) + + return obj +} + +export interface TlSchemaJson { + constructors: TlConstructorJson[] + methods: TlMethodJson[] +} + +export function parseTlSchemaFromJson(obj: object): TlSchemaJson { + assertObject(obj) + + assertFieldType(obj, 'constructors', 'object') + assertFieldType(obj, 'methods', 'object') + + if (!Array.isArray(obj.constructors)) { + throw new Error('Expected field constructors to be an array') + } + + if (!Array.isArray(obj.methods)) { + throw new Error('Expected field methods to be an array') + } + + obj.constructors.forEach(parseTlConstructorFromJson) // will throw if invalid + obj.methods.forEach(parseTlMethodFromJson) // will throw if invalid + + return obj as TlSchemaJson +} diff --git a/packages/tl/scripts/constants.ts b/packages/tl/scripts/constants.ts index 60aab4eb..ab32daff 100644 --- a/packages/tl/scripts/constants.ts +++ b/packages/tl/scripts/constants.ts @@ -20,6 +20,9 @@ export const TDESKTOP_SCHEMA = export const TDESKTOP_LAYER = 'https://raw.githubusercontent.com/telegramdesktop/tdesktop/dev/Telegram/SourceFiles/mtproto/scheme/layer.tl' export const TDLIB_SCHEMA = 'https://raw.githubusercontent.com/tdlib/td/master/td/generate/scheme/telegram_api.tl' +export const WEBK_SCHEMA = 'https://raw.githubusercontent.com/morethanwords/tweb/master/src/scripts/in/schema.json' +export const WEBA_SCHEMA = 'https://raw.githubusercontent.com/Ajaxy/telegram-tt/master/src/lib/gramjs/tl/static/api.tl' +export const WEBA_LAYER = 'https://raw.githubusercontent.com/Ajaxy/telegram-tt/master/src/lib/gramjs/tl/AllTLObjects.js' export const ESM_PRELUDE = `// This file is auto-generated. Do not edit. "use strict"; diff --git a/packages/tl/scripts/fetch-api.ts b/packages/tl/scripts/fetch-api.ts index 0a0b85a9..e27c32b6 100644 --- a/packages/tl/scripts/fetch-api.ts +++ b/packages/tl/scripts/fetch-api.ts @@ -20,6 +20,7 @@ import { TlFullSchema, writeTlEntryToString, } from '@mtcute/tl-utils' +import { parseTlEntriesFromJson } from '@mtcute/tl-utils/json.js' import { __dirname, @@ -31,6 +32,9 @@ import { TDESKTOP_LAYER, TDESKTOP_SCHEMA, TDLIB_SCHEMA, + WEBA_LAYER, + WEBA_SCHEMA, + WEBK_SCHEMA, } from './constants.js' import { applyDocumentation, fetchDocumentation, getCachedDocumentation } from './documentation.js' import { packTlSchema, TlPackedSchema, unpackTlSchema } from './schema.js' @@ -101,6 +105,44 @@ async function fetchCoreSchema(domain = CORE_DOMAIN, name = 'Core'): Promise { + const schema = await fetchRetry(WEBK_SCHEMA) + const json = JSON.parse(schema) as { + layer: number + API: object + } + + let entries = parseTlEntriesFromJson(json.API, { parseMethodTypes: true }) + entries = entries.filter((it) => { + if (it.kind === 'method') { + // json schema doesn't provide info about generics, remove these + return !it.arguments.some((arg) => arg.type === '!X') && it.type !== 'X' + } + + return true + }) + + return { + name: 'WebK', + layer: json.layer, + content: parseFullTlSchema(entries), + } +} + +async function fetchWebaSchema(): Promise { + const [schema, layerFile] = await Promise.all([fetchRetry(WEBA_SCHEMA), fetchRetry(WEBA_LAYER)]) + + // const LAYER = 174; + const version = layerFile.match(/^const LAYER = (\d+);$/m) + if (!version) throw new Error('Layer number not found') + + return { + name: 'WebA', + layer: parseInt(version[1]), + content: tlToFullSchema(schema), + } +} + function input(rl: readline.Interface, q: string): Promise { return new Promise((resolve) => rl.question(q, resolve)) } @@ -182,18 +224,20 @@ async function overrideInt53(schema: TlFullSchema): Promise { async function main() { console.log('Loading schemas...') - const schemas: Schema[] = [ - await fetchTdlibSchema(), - await fetchTdesktopSchema(), - await fetchCoreSchema(), - await fetchCoreSchema(COREFORK_DOMAIN, 'Corefork'), - await fetchCoreSchema(BLOGFORK_DOMAIN, 'Blogfork'), - { + const schemas: Schema[] = await Promise.all([ + fetchTdlibSchema(), + fetchTdesktopSchema(), + fetchCoreSchema(), + fetchCoreSchema(COREFORK_DOMAIN, 'Corefork'), + fetchCoreSchema(BLOGFORK_DOMAIN, 'Blogfork'), + fetchWebkSchema(), + fetchWebaSchema(), + readFile(join(__dirname, '../data/custom.tl'), 'utf8').then((tl) => ({ name: 'Custom', layer: 0, // handled manually - content: tlToFullSchema(await readFile(join(__dirname, '../data/custom.tl'), 'utf8')), - }, - ] + content: tlToFullSchema(tl), + })), + ]) console.log('Available schemas:') schemas.forEach((schema) =>