280c9f51aa
fixed imports, package.json files, improved package generation for certain packages, and more (i'm too lazy to describe all the magic that i've done)
279 lines
8.6 KiB
JavaScript
279 lines
8.6 KiB
JavaScript
const fs = require('fs')
|
|
const path = require('path')
|
|
const { convertTlToJson } = require('../../packages/tl/scripts/generate-schema')
|
|
const fetch = require('node-fetch')
|
|
const qs = require('querystring')
|
|
const { convertToArrays } = require('./prepare-data')
|
|
|
|
const UNIX_0 = '1970-01-01T00:00:00Z'
|
|
const CURRENT_FILE = 'Telegram/Resources/tl/api.tl'
|
|
const FILES = [
|
|
'Telegram/SourceFiles/mtproto/scheme.tl',
|
|
'Telegram/Resources/scheme.tl',
|
|
CURRENT_FILE,
|
|
]
|
|
|
|
async function getLastFetched() {
|
|
return fs.promises
|
|
.readFile(
|
|
path.join(__dirname, '../data/history/last-fetched.txt'),
|
|
'utf8'
|
|
)
|
|
.then((res) => JSON.parse(res))
|
|
.catch(() =>
|
|
FILES.reduce((a, b) => {
|
|
a[b] = UNIX_0
|
|
return a
|
|
}, {})
|
|
)
|
|
}
|
|
|
|
async function updateLastFetched(file, time) {
|
|
return getLastFetched().then((state) =>
|
|
fs.promises.writeFile(
|
|
path.join(__dirname, '../data/history/last-fetched.txt'),
|
|
JSON.stringify({
|
|
...state,
|
|
[file]: time,
|
|
})
|
|
)
|
|
)
|
|
}
|
|
|
|
async function getCounts() {
|
|
return fs.promises
|
|
.readFile(path.join(__dirname, '../data/history/counts.txt'), 'utf8')
|
|
.then((res) => JSON.parse(res))
|
|
.catch(() => ({}))
|
|
}
|
|
|
|
async function setCounts(obj) {
|
|
return fs.promises.writeFile(
|
|
path.join(__dirname, '../data/history/counts.txt'),
|
|
JSON.stringify(obj)
|
|
)
|
|
}
|
|
|
|
async function getFileContent(file, commit) {
|
|
return fetch(
|
|
`https://raw.githubusercontent.com/telegramdesktop/tdesktop/${commit}/${file}`
|
|
).then((r) => r.text())
|
|
}
|
|
|
|
async function parseRemoteTl(file, commit) {
|
|
let content = await getFileContent(file, commit)
|
|
if (content === '404: Not Found') return null
|
|
|
|
let layer = (function () {
|
|
const m = content.match(/^\/\/ LAYER (\d+)/m)
|
|
if (m) return m[1]
|
|
return null
|
|
})()
|
|
if (!layer) {
|
|
// older files did not contain layer number in comment.
|
|
if (content.match(/invokeWithLayer#da9b0d0d/)) {
|
|
// if this is present, then the layer number is available in
|
|
// Telegram/SourceFiles/mtproto/mtpCoreTypes.h
|
|
let mtpCoreTypes = await getFileContent(
|
|
'Telegram/SourceFiles/mtproto/mtpCoreTypes.h',
|
|
commit
|
|
)
|
|
if (mtpCoreTypes === '404: Not Found') {
|
|
mtpCoreTypes = await getFileContent(
|
|
'Telegram/SourceFiles/mtproto/core_types.h',
|
|
commit
|
|
)
|
|
}
|
|
const m = mtpCoreTypes.match(
|
|
/^static const mtpPrime mtpCurrentLayer = (\d+);$/m
|
|
)
|
|
if (!m)
|
|
throw new Error(
|
|
`Could not determine layer number for file ${file} at commit ${commit}`
|
|
)
|
|
layer = m[1]
|
|
} else {
|
|
// even older files on ancient layers
|
|
// layer number is the largest available invokeWithLayerN constructor
|
|
let max = 0
|
|
content.replace(/invokeWithLayer(\d+)#[0-f]+/g, (_, $1) => {
|
|
$1 = parseInt($1)
|
|
if ($1 > max) max = $1
|
|
})
|
|
if (max === 0)
|
|
throw new Error(
|
|
`Could not determine layer number for file ${file} at commit ${commit}`
|
|
)
|
|
layer = max + ''
|
|
}
|
|
}
|
|
|
|
if (content.match(/bad_server_salt#/)) {
|
|
// this is an older file that contained both mtproto and api
|
|
// since we are only interested in api, remove the mtproto part
|
|
const lines = content.split('\n')
|
|
const apiIdx = lines.indexOf('///////// Main application API')
|
|
if (apiIdx === -1)
|
|
throw new Error('Could not find split point for combined file')
|
|
content = lines.slice(apiIdx).join('\n')
|
|
}
|
|
|
|
return {
|
|
layer,
|
|
content,
|
|
tl: convertToArrays(convertTlToJson(content, 'api', true)),
|
|
}
|
|
}
|
|
|
|
function fileSafeDateFormat(date) {
|
|
date = new Date(date)
|
|
return date
|
|
.toISOString()
|
|
.replace(/[\-:]|\.\d\d\d/g, '')
|
|
.split('T')[0]
|
|
}
|
|
|
|
function shortSha(sha) {
|
|
return sha.substr(0, 7)
|
|
}
|
|
|
|
async function fetchHistory(file, since, counts, defaultPrev = null, defaultPrevFile = null) {
|
|
const history = await (async function () {
|
|
const ret = []
|
|
let page = 1
|
|
while (true) {
|
|
const chunk = await fetch(
|
|
`https://api.github.com/repos/telegramdesktop/tdesktop/commits?` +
|
|
qs.stringify({
|
|
since,
|
|
path: file,
|
|
per_page: 100,
|
|
page,
|
|
})
|
|
).then((r) => r.json())
|
|
|
|
if (!chunk.length) break
|
|
|
|
ret.push(...chunk)
|
|
page += 1
|
|
}
|
|
|
|
return ret
|
|
})()
|
|
|
|
// should not happen
|
|
if (history.length === 0) throw new Error('history is empty')
|
|
|
|
const filename = (schema, commit) =>
|
|
`layer${schema.layer}-${fileSafeDateFormat(
|
|
commit.commit.committer.date
|
|
)}-${shortSha(commit.sha)}.json`
|
|
|
|
const uid = (schema, commit) => `${schema.layer}_${shortSha(commit.sha)}`
|
|
|
|
function writeSchemaToFile(schema, commit) {
|
|
return fs.promises.writeFile(
|
|
path.join(__dirname, `../data/history/${filename(schema, commit)}`),
|
|
JSON.stringify({
|
|
// layer is ever-incrementing, sha is random, so no collisions
|
|
uid: uid(schema, commit),
|
|
tl: JSON.stringify(schema.tl),
|
|
layer: parseInt(schema.layer),
|
|
rev:
|
|
schema.layer in counts
|
|
? ++counts[schema.layer]
|
|
: (counts[schema.layer] = 0),
|
|
content: schema.content,
|
|
prev: schema.prev ? schema.prev : defaultPrev,
|
|
prevFile: schema.prevFile ? schema.prevFile : defaultPrevFile,
|
|
source: {
|
|
file,
|
|
date: commit.commit.committer.date,
|
|
commit: commit.sha,
|
|
message: commit.message,
|
|
},
|
|
})
|
|
)
|
|
}
|
|
|
|
let base = history.pop()
|
|
let baseSchema = await parseRemoteTl(file, base.sha)
|
|
let baseFilename = () => filename(baseSchema, base)
|
|
|
|
try {
|
|
await fs.promises.access(
|
|
path.join(__dirname, `../data/history/${baseFilename()}`),
|
|
fs.F_OK
|
|
)
|
|
} catch (e) {
|
|
await writeSchemaToFile(baseSchema, base)
|
|
}
|
|
|
|
while (history.length) {
|
|
const next = history.pop()
|
|
const nextSchema = await parseRemoteTl(file, next.sha)
|
|
if (!nextSchema) break
|
|
|
|
nextSchema.prev = uid(baseSchema, base)
|
|
nextSchema.prevFile = baseFilename()
|
|
base = next
|
|
baseSchema = nextSchema
|
|
await updateLastFetched(file, base.commit.committer.date)
|
|
await setCounts(counts)
|
|
await writeSchemaToFile(baseSchema, base)
|
|
console.log(
|
|
'Fetched commit %s, file %s (%s)',
|
|
shortSha(base.sha),
|
|
file,
|
|
base.commit.committer.date
|
|
)
|
|
}
|
|
|
|
if (file !== CURRENT_FILE) {
|
|
await updateLastFetched(file, `DONE:${uid(baseSchema, base)}:${baseFilename()}`)
|
|
}
|
|
|
|
console.log('No more commits for %s', file)
|
|
}
|
|
|
|
async function main() {
|
|
let last = await getLastFetched()
|
|
const counts = await getCounts()
|
|
|
|
for (let i = 0; i < FILES.length; i++) {
|
|
const file = FILES[i]
|
|
const prev = FILES[i - 1]
|
|
if (!last[file].startsWith('DONE')) {
|
|
let parent = prev ? last[prev].split(':')[1] : null
|
|
let parentFile = prev ? last[prev].split(':')[2] : null
|
|
|
|
await fetchHistory(file, last[file], counts, parent, parentFile)
|
|
|
|
last = await getLastFetched()
|
|
}
|
|
}
|
|
|
|
console.log('Creating reverse links ("next" field)')
|
|
for (const file of await fs.promises.readdir(
|
|
path.join(__dirname, '../data/history')
|
|
)) {
|
|
if (!file.startsWith('layer')) continue
|
|
|
|
const fullPath = path.join(__dirname, '../data/history', file)
|
|
const json = JSON.parse(await fs.promises.readFile(fullPath, 'utf-8'))
|
|
if (json.prev) {
|
|
const parentPath = path.join(
|
|
__dirname,
|
|
'../data/history',
|
|
json.prevFile
|
|
)
|
|
const parentJson = JSON.parse(
|
|
await fs.promises.readFile(parentPath, 'utf-8')
|
|
)
|
|
parentJson.next = json.uid
|
|
await fs.promises.writeFile(parentPath, JSON.stringify(parentJson))
|
|
}
|
|
}
|
|
}
|
|
|
|
main().catch(console.error)
|