teidesu/mtcute
1
0
Fork 0
Code Issues 15 Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat(tl): merge schemas from tdlib and tdesktop

Browse source
This commit is contained in:
teidesu 2021-06-26 16:22:19 +03:00
parent cf7f8e74ea
commit 1abfc56474
3 changed files with 524 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
packages/tl/scripts/generate-schema.js
View file

@ -12,6 +12,9 @@ const { applyDescriptionsFile } = require('./process-descriptions-yaml')
const yaml = require('js-yaml')
const { snakeToCamel } = require('./common')
const { asyncPool } = require('eager-async-pool')
const { mergeSchemas } = require('./merge-schemas')
const CRC32 = require('crc-32')
const SingleRegex = /^(.+?)(?:#([0-f]{1,8}))?(?: \?)?(?: {(.+?:.+?)})? ((?:.+? )*)= (.+);$/
const transformIgnoreNamespace = (fn, s) => {
@ -163,13 +166,24 @@ function convertTlToJson(tlText, tlType, silent = false) {
if (!match) {
console.warn('Regex failed on:\n"' + line + '"')
} else {
let [, fullName, typeId = '0', generics, args, type] = match
let [, fullName, typeId, generics, args, type] = match
if (fullName in _types || fullName === 'vector') {
// vector is parsed manually
nextLine()
continue
}
if (!typeId) {
typeId = CRC32.str(
// normalize
line
.replace(/[{}]|[a-zA-Z0-9_]+:flags\.[0-9]+\?true/g, '')
.replace(/[<>]/g, ' ')
.replace(/ +/g, ' ')
.trim()
)
}
args = args.trim()
args =
args && !args.match(/\[ [a-z]+ ]/i)
@ -496,12 +510,41 @@ async function main() {
ret.mtproto = convertTlToJson(mtprotoTl, 'mtproto')
console.log('[i] Fetching api.tl')
let apiTl = await fetch(
console.log('[i] Fetching api.tl from tdesktop')
const apiTlDesktop = await fetch(
'https://raw.githubusercontent.com/telegramdesktop/tdesktop/dev/Telegram/Resources/tl/api.tl'
).then((i) => i.text())
ret.apiLayer = apiTl.match(/^\/\/ LAYER (\d+)/m)[1]
ret.api = convertTlToJson(apiTl, 'api')
const apiDesktopLayer = parseInt(apiTlDesktop.match(/^\/\/ LAYER (\d+)/m)[1])
console.log('[i] Fetching telegram_api.tl from TDLib')
const apiTlTdlib = await fetch(
'https://raw.githubusercontent.com/tdlib/td/master/td/generate/scheme/telegram_api.tl'
).then((i) => i.text())
const apiTdlibLayer = await fetch('https://raw.githubusercontent.com/tdlib/td/master/td/telegram/Version.h')
.then((r) => r.text())
.then((res) => parseInt(res.match(/^constexpr int32 MTPROTO_LAYER = (\d+)/m)[1]))
console.log('[i] tdesktop has layer %d, tdlib has %d', apiDesktopLayer, apiTdlibLayer)
if (Math.abs(apiDesktopLayer - apiTdlibLayer) > 2) {
console.log('[i] Too different layers, using newer one')
const newer = apiDesktopLayer > apiTdlibLayer ? apiTlDesktop : apiTlTdlib
const newerLayer = apiDesktopLayer > apiTdlibLayer ? apiDesktopLayer : apiTdlibLayer
ret.apiLayer = newerLayer + ''
ret.api = convertTlToJson(newer, 'api')
} else {
console.log('[i] Merging schemas...')
const first = convertTlToJson(apiTlTdlib, 'api')
const second = convertTlToJson(apiTlDesktop, 'api')
await mergeSchemas(first, second)
ret.apiLayer = apiTdlibLayer + ''
ret.api = first
}
await addDocumentation(ret.api)
await applyDescriptionsFile(ret, descriptionsYaml)

458
packages/tl/scripts/merge-schemas.js Normal file
View file

@ -0,0 +1,458 @@
// used by generate-schema, but since logic is quite large, moved it to a separate file
const CRC32 = require('crc-32')
const signedInt32ToUnsigned = (val) => (val < 0 ? val + 0x100000000 : val)
// converting map from custom type back to tl
const _types = {
number: 'int',
Long: 'long',
Int128: 'int128',
Int256: 'int256',
Double: 'double',
string: 'string',
Buffer: 'bytes',
false: 'boolFalse',
true: 'boolTrue',
boolean: 'bool',
boolean: 'Bool',
true: 'true',
null: 'null',
any: 'Type',
$FlagsBitField: '#',
}
function convertType(typ) {
if (typ in _types) return _types[typ]
let m = typ.match(/^(.+?)\[\]$/)
if (m) {
return 'Vector ' + convertType(m[1])
}
return typ
}
function stringifyType(cls, ns = cls._ns, includeId = true) {
let str = ''
if (ns !== '$root') {
str += ns + '.'
}
str += cls.name
if (includeId && cls.id) {
str += '#' + cls.id.toString(16)
}
str += ' '
if (cls.generics) {
for (const g of cls.generics) {
str += g.name + ':' + convertType(g.super) + ' '
}
}
for (const arg of cls.arguments) {
if (arg.optional && arg.type === 'true') continue
str += arg.name + ':'
if (arg.optional) {
str += arg.predicate + '?'
}
if (arg.type === 'X') {
str += '!'
}
str += convertType(arg.type) + ' '
}
str += '= ' + convertType(cls.type || cls.returns)
return str
}
function computeConstructorId(ns, cls) {
return signedInt32ToUnsigned(CRC32.str(stringifyType(cls, ns, false)))
}
function createTlSchemaIndex(schema) {
let ret = {}
Object.entries(schema).forEach(([ns, it]) => {
it.classes.forEach((obj) => {
obj.uid = 'c_' + ns + '.' + obj.name
obj._ns = ns
obj._type = 'classes'
ret[obj.uid] = obj
})
it.methods.forEach((obj) => {
obj.uid = 'm_' + ns + '.' + obj.name
obj._ns = ns
obj._type = 'methods'
ret[obj.uid] = obj
})
it.unions.forEach((obj) => {
obj.uid = 'u_' + ns + '.' + obj.type
obj._ns = ns
obj._type = 'unions'
ret[obj.uid] = obj
})
})
return ret
}
// merge schema `b` into `a` (does not copy)
async function mergeSchemas(a, b) {
const rl = require('readline').createInterface({
input: process.stdin,
output: process.stdout,
})
const input = (q) => new Promise((res) => rl.question(q, res))
const index = createTlSchemaIndex(a)
const indexB = createTlSchemaIndex(b)
for (const [uid, objB] of Object.entries(indexB)) {
if (!(uid in index)) {
// just add
index[uid] = objB
if (!a[objB._ns])
a[objB._ns] = {
classes: [],
methods: [],
unions: [],
}
if (!a[objB._ns][objB._type]) a[objB._ns][objB._type] = []
a[objB._ns][objB._type].push(objB)
continue
}
const objA = index[uid]
if (objB._type === 'unions') {
// merge subclasses
objA.subtypes = [...new Set([...objA.subtypes, ...objB.subtypes])]
continue
}
// check for conflict
if (objA.id !== objB.id) {
console.log('! CONFLICT !')
console.log('Schema A (tdlib): %s', stringifyType(objA))
console.log('Schema B (tdesktop): %s', stringifyType(objB))
let keep
while (true) {
keep = await input('Which to keep? [A/B] > ')
keep = keep.toUpperCase()
if (keep !== 'A' && keep !== 'B') {
console.log('Invalid input! Please type A or B')
continue
}
break
}
if (keep === 'B') {
index[objB.uid] = objB
const idx = a[objB._ns][objB._type].findIndex((it) => it.uid === objB.uid)
a[objB._ns][objB._type][idx] = objB
}
continue
}
// now ctor id is the same, meaning that only `true` flags may differ.
// merge them.
const argsIndex = {}
objA.arguments.forEach((arg) => {
argsIndex[arg.name] = arg
})
objB.arguments.forEach((arg) => {
if (!(arg.name in argsIndex)) {
objA.arguments.push(arg)
}
})
}
// clean up
Object.values(index).forEach((obj) => {
delete obj.uid
delete obj._ns
delete obj._type
})
rl.close()
}
module.exports = { mergeSchemas }
if (require.main === module) {
const { expect } = require('chai')
const schema = require('../raw-schema.json')
console.log('testing ctor id computation')
Object.entries(schema.api, (ns, items) => {
for (const obj of items.methods) {
if (obj.id !== computeConstructorId(ns, obj)) {
console.log('invalid ctor id: %s', obj.name)
}
}
for (const obj of items.classes) {
if (obj.id !== computeConstructorId(ns, obj)) {
console.log('invalid ctor id: %s', obj.name)
}
}
})
async function test() {
function makeNamespace (obj = {}) {
return {
classes: [],
methods: [],
unions: [],
...obj
}
}
async function testMergeSchemas (name, a, b, expected) {
await mergeSchemas(a, b)
expect(a).eql(expected, name)
}
console.log('testing merging')
await testMergeSchemas('new type', {
test: makeNamespace()
}, {
test: makeNamespace({
methods: [
{
name: 'useError',
returns: 'Error',
arguments: []
}
]
})
}, {
test: makeNamespace({
methods: [
{
name: 'useError',
returns: 'Error',
arguments: []
}
]
})
})
await testMergeSchemas('different union', {
help: makeNamespace({
unions: [
{
type: 'ConfigSimple',
subtypes: [
'testA',
'testB'
]
}
]
})
}, {
help: makeNamespace({
unions: [
{
type: 'ConfigSimple',
subtypes: [
'testB',
'testC'
]
}
]
})
}, {
help: makeNamespace({
unions: [
{
type: 'ConfigSimple',
subtypes: [
'testA',
'testB',
'testC'
]
}
]
})
})
await testMergeSchemas('different class', {
help: makeNamespace({
classes: [
{
name: 'configSimple',
type: 'ConfigSimple',
arguments: [
{
name: 'flags',
type: '$FlagsBitField'
},
{
name: 'date',
type: 'number'
},
{
name: 'includeThis',
optional: true,
predicate: 'flags.0',
type: 'true'
}
]
}
]
})
}, {
help: makeNamespace({
classes: [
{
name: 'configSimple',
type: 'ConfigSimple',
arguments: [
{
name: 'flags',
type: '$FlagsBitField'
},
{
name: 'date',
type: 'number'
},
{
name: 'includeThat',
optional: true,
predicate: 'flags.0',
type: 'true'
}
]
}
]
})
}, {
help: makeNamespace({
classes: [
{
name: 'configSimple',
type: 'ConfigSimple',
arguments: [
{
name: 'flags',
type: '$FlagsBitField'
},
{
name: 'date',
type: 'number'
},
{
name: 'includeThis',
optional: true,
predicate: 'flags.0',
type: 'true'
},
{
name: 'includeThat',
optional: true,
predicate: 'flags.0',
type: 'true'
}
]
}
]
})
})
function addId(ns, obj) {
obj.id = computeConstructorId(ns, obj)
return obj
}
console.log('vv choose B vv')
await testMergeSchemas('conflicting class', {
help: makeNamespace({
classes: [
addId('help', {
name: 'configSimple',
type: 'ConfigSimple',
arguments: [
{
name: 'flags',
type: '$FlagsBitField'
},
{
name: 'date',
type: 'number'
}
]
})
]
})
}, {
help: makeNamespace({
classes: [
addId('help', {
name: 'configSimple',
type: 'ConfigSimple',
arguments: [
{
name: 'flags',
type: '$FlagsBitField'
},
{
name: 'date',
type: 'number'
},
{
name: 'expires',
type: 'number'
}
]
})
]
})
}, {
help: makeNamespace({
classes: [
addId('help', {
name: 'configSimple',
type: 'ConfigSimple',
arguments: [
{
name: 'flags',
type: '$FlagsBitField'
},
{
name: 'date',
type: 'number'
},
{
name: 'expires',
type: 'number'
}
]
})
]
})
})
}
test().catch(console.error)
}

18
yarn.lock
View file

@ -2117,6 +2117,14 @@ cosmiconfig@^7.0.0:
path-type "^4.0.0"
yaml "^1.10.0"
crc-32@^1.2.0:
version "1.2.0"
resolved "http://localhost:4873/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208"
integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==
dependencies:
exit-on-epipe "~1.0.1"
printj "~1.1.0"
create-require@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
@ -2666,6 +2674,11 @@ execa@^5.0.0:
signal-exit "^3.0.3"
strip-final-newline "^2.0.0"
exit-on-epipe@~1.0.1:
version "1.0.1"
resolved "http://localhost:4873/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692"
integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==
expand-template@^2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c"
@ -5074,6 +5087,11 @@ prettier@2.2.1:
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5"
integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==
printj@~1.1.0:
version "1.1.2"
resolved "http://localhost:4873/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222"
integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==
process-nextick-args@~2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"