mtcute/packages/tl/scripts/documentation.ts

import { TlEntry, TlFullSchema } from '@mtcute/tl-utils/src/types'
import cheerio from 'cheerio'
import { splitNameToNamespace } from '@mtcute/tl-utils/src/utils'
import { snakeToCamel } from '@mtcute/tl-utils/src/codegen/utils'
import {
    API_SCHEMA_JSON_FILE,
    CORE_DOMAIN,
    DESCRIPTIONS_YAML_FILE,
    DOC_CACHE_FILE,
} from './constants'
import { fetchRetry } from './utils'
import { readFile, writeFile } from 'fs/promises'
// @ts-ignore
import jsYaml from 'js-yaml'
import { applyDescriptionsYamlFile } from './process-descriptions-yaml'
import { packTlSchema, unpackTlSchema } from './schema'

type Cheerio = typeof cheerio['root'] extends () => infer T ? T : never
type CheerioInit = typeof cheerio['load'] extends (...a: any[]) => infer T
    ? T
    : never

export interface CachedDocumentationEntry {
    comment?: string
    arguments?: Record<string, string>
    throws?: TlEntry['throws']
    available?: TlEntry['available']
}

export interface CachedDocumentation {
    updated: string
    classes: Record<string, CachedDocumentationEntry>
    methods: Record<string, CachedDocumentationEntry>
    unions: Record<string, string>
}

function normalizeLinks(url: string, el: Cheerio): void {
    el.find('a').each((i, _it) => {
        const it = cheerio(_it)

        if (it.attr('href')![0] === '#') return

        it.attr('href', new URL(it.attr('href')!, url).href)
        let href = it.attr('href')!

        let m
        if (
            (m = href.match(/\/(constructor|method|union)\/([^#?]+)(?:\?|#|$)/))
        ) {
            let [, type, name] = m
            if (type === 'method') {
                const [ns, n] = splitNameToNamespace(name)
                const q = snakeToCamel(n)
                name = ns ? ns + '.' + q : q
            }

            it.replaceWith(`{@link ${name}}`)
        }
    })
}

function extractDescription($: CheerioInit) {
    return $('.page_scheme')
        .prevAll('p')
        .get()
        .reverse()
        .map((el) => $(el).html()!.trim())
        .join('\n\n')
        .trim()
}

// from https://github.com/sindresorhus/cli-spinners/blob/main/spinners.json
const PROGRESS_CHARS = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']

export async function fetchDocumentation(
    schema: TlFullSchema,
    layer: number,
    silent = !process.stdout.isTTY
): Promise<CachedDocumentation> {
    const headers = {
        cookie: `stel_dev_layer=${layer}`,
        'User-Agent':
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
            'Chrome/87.0.4280.88 Safari/537.36',
    }

    const ret: CachedDocumentation = {
        updated: `${new Date().toLocaleString('ru-RU')} (layer ${layer})`,
        classes: {},
        methods: {},
        unions: {},
    }

    let prevSize = 0
    let logPos = 0

    function log(str: string) {
        if (silent) return
        while (str.length < prevSize) str += ' '

        process.stdout.write('\r' + PROGRESS_CHARS[logPos] + ' ' + str)

        prevSize = str.length
        logPos = (logPos + 1) % PROGRESS_CHARS.length
    }

    for (const entry of schema.entries) {
        log(`📥 ${entry.kind} ${entry.name}`)

        const url = `${CORE_DOMAIN}/${
            entry.kind === 'class' ? 'constructor' : 'method'
        }/${entry.name}`

        const html = await fetchRetry(url, {
            headers,
        })
        const $ = cheerio.load(html)
        const content = $('#dev_page_content')

        if (content.text().trim() === 'The page has not been saved') continue

        normalizeLinks(url, content)

        const retClass: CachedDocumentationEntry = {}

        const description = extractDescription($)

        if (description) {
            retClass.comment = description
        }

        const parametersTable = $('#parameters').parent().next('table')
        parametersTable.find('tr').each((idx, _el) => {
            const el = $(_el)
            const cols = el.find('td')
            if (!cols.length) return // <thead>

            const name = snakeToCamel(cols.first().text().trim())
            const description = cols.last().html()!.trim()

            if (description) {
                if (!retClass.arguments) retClass.arguments = {}
                retClass.arguments[name] = description
            }
        })

        if (entry.kind === 'method') {
            const errorsTable = $('#possible-errors').parent().next('table')

            let userBotRequired = false

            errorsTable.find('tr').each((idx, _el) => {
                const el = $(_el)
                let cols = el.find('td')
                if (!cols.length) return // <thead>

                let code = parseInt($(cols[0]).text())
                let name = $(cols[1]).text()
                let comment = $(cols[2]).text()

                if (name === 'USER_BOT_REQUIRED') userBotRequired = true

                if (!retClass.throws) retClass.throws = []
                retClass.throws.push({ code, name, comment })
            })

            const botsCanUse = !!$('#bots-can-use-this-method').length
            const onlyBotsCanUse =
                botsCanUse &&
                (!!description.match(/[,;]( for)? bots only$/) ||
                    userBotRequired)

            retClass.available = onlyBotsCanUse
                ? 'bot'
                : botsCanUse
                ? 'both'
                : 'user'
        }

        ret[entry.kind === 'class' ? 'classes' : 'methods'][
            entry.name
        ] = retClass
    }

    for (const name in schema.unions) {
        if (!schema.unions.hasOwnProperty(name)) continue

        log(`📥 union ${name}`)

        const url = `${CORE_DOMAIN}/type/${name}`

        const html = await fetchRetry(url, {
            headers,
        })
        const $ = cheerio.load(html)
        const content = $('#dev_page_content')

        if (content.text().trim() === 'The page has not been saved') continue

        normalizeLinks(url, content)

        const description = extractDescription($)
        if (description) ret.unions[name] = description
    }

    log('✨ Patching descriptions')

    const descriptionsYaml = jsYaml.load(
        await readFile(DESCRIPTIONS_YAML_FILE, 'utf8')
    )
    applyDescriptionsYamlFile(ret, descriptionsYaml)

    log('🔄 Writing to file')

    await writeFile(DOC_CACHE_FILE, JSON.stringify(ret))

    if (!silent) process.stdout.write('\n')

    return ret
}

export function applyDocumentation(
    schema: TlFullSchema,
    docs: CachedDocumentation
) {
    for (let i = 0; i < 2; i++) {
        const kind = i === 0 ? 'classes' : 'methods'

        const objIndex = schema[kind]
        const docIndex = docs[kind]

        for (let name in docIndex) {
            if (!docIndex.hasOwnProperty(name)) continue
            if (!(name in objIndex)) continue

            const obj = objIndex[name]
            const doc = docIndex[name]

            if (doc.comment) obj.comment = doc.comment
            if (doc.throws) obj.throws = doc.throws
            if (doc.available) obj.available = doc.available
            if (doc.arguments) {
                obj.arguments.forEach((arg) => {
                    if (arg.name in doc.arguments!) {
                        arg.comment = doc.arguments![arg.name]
                    }
                })
            }
        }
    }

    for (let name in schema.unions) {
        if (!schema.unions.hasOwnProperty(name)) continue
        if (!(name in docs.unions)) continue

        schema.unions[name].comment = docs.unions[name]
    }
}

export async function getCachedDocumentation(): Promise<CachedDocumentation | null> {
    try {
        const file = await readFile(DOC_CACHE_FILE, 'utf8')
        return JSON.parse(file)
    } catch (e: any) {
        if (e.code === 'ENOENT') {
            return null
        }
        throw e
    }
}

async function main() {
    let cached = await getCachedDocumentation()

    if (cached) {
        console.log('Cached documentation: %d', cached.updated)
    }

    const rl = require('readline').createInterface({
        input: process.stdin,
        output: process.stdout,
    })
    const input = (q: string): Promise<string> =>
        new Promise((res) => rl.question(q, res))

    while (true) {
        console.log('Choose action:')
        console.log('0. Exit')
        console.log('1. Update documentation')
        console.log('2. Apply descriptions.yaml')
        console.log('3. Apply documentation to schema')

        const act = parseInt(await input('[0-3] > '))
        if (isNaN(act) || act < 0 || act > 3) {
            console.log('Invalid action')
            continue
        }

        if (act === 0) return

        if (act === 1) {
            const [schema, layer] = unpackTlSchema(
                JSON.parse(await readFile(API_SCHEMA_JSON_FILE, 'utf8'))
            )
            cached = await fetchDocumentation(schema, layer)
        }

        if (act === 2) {
            if (!cached) {
                console.log('No schema available, fetch it first')
                continue
            }

            const descriptionsYaml = jsYaml.load(
                await readFile(DESCRIPTIONS_YAML_FILE, 'utf8')
            )
            applyDescriptionsYamlFile(cached, descriptionsYaml)

            await writeFile(DOC_CACHE_FILE, JSON.stringify(cached))
        }

        if (act === 3) {
            if (!cached) {
                console.log('No schema available, fetch it first')
                continue
            }

            const [schema, layer] = unpackTlSchema(
                JSON.parse(await readFile(API_SCHEMA_JSON_FILE, 'utf8'))
            )

            applyDocumentation(schema, cached)
            await writeFile(
                API_SCHEMA_JSON_FILE,
                JSON.stringify(packTlSchema(schema, layer))
            )
        }
    }
}

if (require.main === module) {
    main().catch(console.error)
}