From 1abfc564740aa9e70f753c21e5e0c50a0d34e72a Mon Sep 17 00:00:00 2001 From: teidesu <86301490+teidesu@users.noreply.github.com> Date: Sat, 26 Jun 2021 16:22:19 +0300 Subject: [PATCH] feat(tl): merge schemas from tdlib and tdesktop --- packages/tl/scripts/generate-schema.js | 53 ++- packages/tl/scripts/merge-schemas.js | 458 +++++++++++++++++++++++++ yarn.lock | 18 + 3 files changed, 524 insertions(+), 5 deletions(-) create mode 100644 packages/tl/scripts/merge-schemas.js diff --git a/packages/tl/scripts/generate-schema.js b/packages/tl/scripts/generate-schema.js index 35d3886f..a75288ee 100644 --- a/packages/tl/scripts/generate-schema.js +++ b/packages/tl/scripts/generate-schema.js @@ -12,6 +12,9 @@ const { applyDescriptionsFile } = require('./process-descriptions-yaml') const yaml = require('js-yaml') const { snakeToCamel } = require('./common') const { asyncPool } = require('eager-async-pool') +const { mergeSchemas } = require('./merge-schemas') +const CRC32 = require('crc-32') + const SingleRegex = /^(.+?)(?:#([0-f]{1,8}))?(?: \?)?(?: {(.+?:.+?)})? ((?:.+? )*)= (.+);$/ const transformIgnoreNamespace = (fn, s) => { @@ -163,13 +166,24 @@ function convertTlToJson(tlText, tlType, silent = false) { if (!match) { console.warn('Regex failed on:\n"' + line + '"') } else { - let [, fullName, typeId = '0', generics, args, type] = match + let [, fullName, typeId, generics, args, type] = match if (fullName in _types || fullName === 'vector') { // vector is parsed manually nextLine() continue } + if (!typeId) { + typeId = CRC32.str( + // normalize + line + .replace(/[{}]|[a-zA-Z0-9_]+:flags\.[0-9]+\?true/g, '') + .replace(/[<>]/g, ' ') + .replace(/ +/g, ' ') + .trim() + ) + } + args = args.trim() args = args && !args.match(/\[ [a-z]+ ]/i) @@ -496,12 +510,41 @@ async function main() { ret.mtproto = convertTlToJson(mtprotoTl, 'mtproto') - console.log('[i] Fetching api.tl') - let apiTl = await fetch( + console.log('[i] Fetching api.tl from tdesktop') + const apiTlDesktop = await fetch( 'https://raw.githubusercontent.com/telegramdesktop/tdesktop/dev/Telegram/Resources/tl/api.tl' ).then((i) => i.text()) - ret.apiLayer = apiTl.match(/^\/\/ LAYER (\d+)/m)[1] - ret.api = convertTlToJson(apiTl, 'api') + const apiDesktopLayer = parseInt(apiTlDesktop.match(/^\/\/ LAYER (\d+)/m)[1]) + + console.log('[i] Fetching telegram_api.tl from TDLib') + const apiTlTdlib = await fetch( + 'https://raw.githubusercontent.com/tdlib/td/master/td/generate/scheme/telegram_api.tl' + ).then((i) => i.text()) + const apiTdlibLayer = await fetch('https://raw.githubusercontent.com/tdlib/td/master/td/telegram/Version.h') + .then((r) => r.text()) + .then((res) => parseInt(res.match(/^constexpr int32 MTPROTO_LAYER = (\d+)/m)[1])) + + console.log('[i] tdesktop has layer %d, tdlib has %d', apiDesktopLayer, apiTdlibLayer) + + if (Math.abs(apiDesktopLayer - apiTdlibLayer) > 2) { + console.log('[i] Too different layers, using newer one') + + const newer = apiDesktopLayer > apiTdlibLayer ? apiTlDesktop : apiTlTdlib + const newerLayer = apiDesktopLayer > apiTdlibLayer ? apiDesktopLayer : apiTdlibLayer + + ret.apiLayer = newerLayer + '' + ret.api = convertTlToJson(newer, 'api') + } else { + console.log('[i] Merging schemas...') + + const first = convertTlToJson(apiTlTdlib, 'api') + const second = convertTlToJson(apiTlDesktop, 'api') + await mergeSchemas(first, second) + + ret.apiLayer = apiTdlibLayer + '' + ret.api = first + } + await addDocumentation(ret.api) await applyDescriptionsFile(ret, descriptionsYaml) diff --git a/packages/tl/scripts/merge-schemas.js b/packages/tl/scripts/merge-schemas.js new file mode 100644 index 00000000..71cd7f94 --- /dev/null +++ b/packages/tl/scripts/merge-schemas.js @@ -0,0 +1,458 @@ +// used by generate-schema, but since logic is quite large, moved it to a separate file +const CRC32 = require('crc-32') + +const signedInt32ToUnsigned = (val) => (val < 0 ? val + 0x100000000 : val) + +// converting map from custom type back to tl +const _types = { + number: 'int', + Long: 'long', + Int128: 'int128', + Int256: 'int256', + Double: 'double', + string: 'string', + Buffer: 'bytes', + false: 'boolFalse', + true: 'boolTrue', + boolean: 'bool', + boolean: 'Bool', + true: 'true', + null: 'null', + any: 'Type', + $FlagsBitField: '#', +} + +function convertType(typ) { + if (typ in _types) return _types[typ] + let m = typ.match(/^(.+?)\[\]$/) + if (m) { + return 'Vector ' + convertType(m[1]) + } + return typ +} + +function stringifyType(cls, ns = cls._ns, includeId = true) { + let str = '' + + if (ns !== '$root') { + str += ns + '.' + } + + str += cls.name + + if (includeId && cls.id) { + str += '#' + cls.id.toString(16) + } + + str += ' ' + + if (cls.generics) { + for (const g of cls.generics) { + str += g.name + ':' + convertType(g.super) + ' ' + } + } + + for (const arg of cls.arguments) { + if (arg.optional && arg.type === 'true') continue + + str += arg.name + ':' + + if (arg.optional) { + str += arg.predicate + '?' + } + + if (arg.type === 'X') { + str += '!' + } + + str += convertType(arg.type) + ' ' + } + + str += '= ' + convertType(cls.type || cls.returns) + + return str +} + +function computeConstructorId(ns, cls) { + return signedInt32ToUnsigned(CRC32.str(stringifyType(cls, ns, false))) +} + +function createTlSchemaIndex(schema) { + let ret = {} + Object.entries(schema).forEach(([ns, it]) => { + it.classes.forEach((obj) => { + obj.uid = 'c_' + ns + '.' + obj.name + obj._ns = ns + obj._type = 'classes' + ret[obj.uid] = obj + }) + it.methods.forEach((obj) => { + obj.uid = 'm_' + ns + '.' + obj.name + obj._ns = ns + obj._type = 'methods' + ret[obj.uid] = obj + }) + it.unions.forEach((obj) => { + obj.uid = 'u_' + ns + '.' + obj.type + obj._ns = ns + obj._type = 'unions' + ret[obj.uid] = obj + }) + }) + return ret +} + +// merge schema `b` into `a` (does not copy) +async function mergeSchemas(a, b) { + const rl = require('readline').createInterface({ + input: process.stdin, + output: process.stdout, + }) + + const input = (q) => new Promise((res) => rl.question(q, res)) + + const index = createTlSchemaIndex(a) + const indexB = createTlSchemaIndex(b) + + for (const [uid, objB] of Object.entries(indexB)) { + if (!(uid in index)) { + // just add + index[uid] = objB + + if (!a[objB._ns]) + a[objB._ns] = { + classes: [], + methods: [], + unions: [], + } + + if (!a[objB._ns][objB._type]) a[objB._ns][objB._type] = [] + + a[objB._ns][objB._type].push(objB) + continue + } + + const objA = index[uid] + + if (objB._type === 'unions') { + // merge subclasses + objA.subtypes = [...new Set([...objA.subtypes, ...objB.subtypes])] + continue + } + + // check for conflict + if (objA.id !== objB.id) { + console.log('! CONFLICT !') + console.log('Schema A (tdlib): %s', stringifyType(objA)) + console.log('Schema B (tdesktop): %s', stringifyType(objB)) + + let keep + while (true) { + keep = await input('Which to keep? [A/B] > ') + keep = keep.toUpperCase() + + if (keep !== 'A' && keep !== 'B') { + console.log('Invalid input! Please type A or B') + continue + } + + break + } + + if (keep === 'B') { + index[objB.uid] = objB + + const idx = a[objB._ns][objB._type].findIndex((it) => it.uid === objB.uid) + a[objB._ns][objB._type][idx] = objB + } + + continue + } + + // now ctor id is the same, meaning that only `true` flags may differ. + // merge them. + + const argsIndex = {} + + objA.arguments.forEach((arg) => { + argsIndex[arg.name] = arg + }) + + objB.arguments.forEach((arg) => { + if (!(arg.name in argsIndex)) { + objA.arguments.push(arg) + } + }) + } + + // clean up + Object.values(index).forEach((obj) => { + delete obj.uid + delete obj._ns + delete obj._type + }) + + rl.close() +} + +module.exports = { mergeSchemas } + +if (require.main === module) { + const { expect } = require('chai') + const schema = require('../raw-schema.json') + + console.log('testing ctor id computation') + Object.entries(schema.api, (ns, items) => { + for (const obj of items.methods) { + if (obj.id !== computeConstructorId(ns, obj)) { + console.log('invalid ctor id: %s', obj.name) + } + } + for (const obj of items.classes) { + if (obj.id !== computeConstructorId(ns, obj)) { + console.log('invalid ctor id: %s', obj.name) + } + } + }) + + + async function test() { + function makeNamespace (obj = {}) { + return { + classes: [], + methods: [], + unions: [], + ...obj + } + } + + async function testMergeSchemas (name, a, b, expected) { + await mergeSchemas(a, b) + + expect(a).eql(expected, name) + } + + console.log('testing merging') + + await testMergeSchemas('new type', { + test: makeNamespace() + }, { + test: makeNamespace({ + methods: [ + { + name: 'useError', + returns: 'Error', + arguments: [] + } + ] + }) + }, { + test: makeNamespace({ + methods: [ + { + name: 'useError', + returns: 'Error', + arguments: [] + } + ] + }) + }) + + await testMergeSchemas('different union', { + help: makeNamespace({ + unions: [ + { + type: 'ConfigSimple', + subtypes: [ + 'testA', + 'testB' + ] + } + ] + }) + }, { + help: makeNamespace({ + unions: [ + { + type: 'ConfigSimple', + subtypes: [ + 'testB', + 'testC' + ] + } + ] + }) + }, { + help: makeNamespace({ + unions: [ + { + type: 'ConfigSimple', + subtypes: [ + 'testA', + 'testB', + 'testC' + ] + } + ] + }) + }) + + await testMergeSchemas('different class', { + help: makeNamespace({ + classes: [ + { + name: 'configSimple', + type: 'ConfigSimple', + arguments: [ + { + name: 'flags', + type: '$FlagsBitField' + }, + { + name: 'date', + type: 'number' + }, + { + name: 'includeThis', + optional: true, + predicate: 'flags.0', + type: 'true' + } + ] + } + ] + }) + }, { + help: makeNamespace({ + classes: [ + { + name: 'configSimple', + type: 'ConfigSimple', + arguments: [ + { + name: 'flags', + type: '$FlagsBitField' + }, + { + name: 'date', + type: 'number' + }, + { + name: 'includeThat', + optional: true, + predicate: 'flags.0', + type: 'true' + } + ] + } + ] + }) + }, { + help: makeNamespace({ + classes: [ + { + name: 'configSimple', + type: 'ConfigSimple', + arguments: [ + { + name: 'flags', + type: '$FlagsBitField' + }, + { + name: 'date', + type: 'number' + }, + { + name: 'includeThis', + optional: true, + predicate: 'flags.0', + type: 'true' + }, + { + name: 'includeThat', + optional: true, + predicate: 'flags.0', + type: 'true' + } + ] + } + ] + }) + }) + + function addId(ns, obj) { + obj.id = computeConstructorId(ns, obj) + return obj + } + + console.log('vv choose B vv') + await testMergeSchemas('conflicting class', { + help: makeNamespace({ + classes: [ + addId('help', { + name: 'configSimple', + type: 'ConfigSimple', + arguments: [ + { + name: 'flags', + type: '$FlagsBitField' + }, + { + name: 'date', + type: 'number' + } + ] + }) + ] + }) + }, { + help: makeNamespace({ + classes: [ + addId('help', { + name: 'configSimple', + type: 'ConfigSimple', + arguments: [ + { + name: 'flags', + type: '$FlagsBitField' + }, + { + name: 'date', + type: 'number' + }, + { + name: 'expires', + type: 'number' + } + ] + }) + ] + }) + }, { + help: makeNamespace({ + classes: [ + addId('help', { + name: 'configSimple', + type: 'ConfigSimple', + arguments: [ + { + name: 'flags', + type: '$FlagsBitField' + }, + { + name: 'date', + type: 'number' + }, + { + name: 'expires', + type: 'number' + } + ] + }) + ] + }) + }) + } + + test().catch(console.error) +} diff --git a/yarn.lock b/yarn.lock index 05699ec3..f3a53dd3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2117,6 +2117,14 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +crc-32@^1.2.0: + version "1.2.0" + resolved "http://localhost:4873/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208" + integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA== + dependencies: + exit-on-epipe "~1.0.1" + printj "~1.1.0" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -2666,6 +2674,11 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" +exit-on-epipe@~1.0.1: + version "1.0.1" + resolved "http://localhost:4873/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" + integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw== + expand-template@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" @@ -5074,6 +5087,11 @@ prettier@2.2.1: resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5" integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q== +printj@~1.1.0: + version "1.1.2" + resolved "http://localhost:4873/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" + integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== + process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"