feat(tl): updated layer 166

gotta love telegram versioning
also made docs downloading faster

packages/tl/README.md

@@ -2,7 +2,7 @@
 
 TL schema and related utils used for mtcute.
 
-Generated from TL layer **166** (last updated on 29.10.2023).
+Generated from TL layer **166** (last updated on 17.11.2023).
 
 ## About
 

packages/tl/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@mtcute/tl",
-    "version": "166.0.0",
+    "version": "166.1.0",
     "description": "TL schema used for mtcute",
     "main": "index.js",
     "author": "Alina Sireneva <alina@tei.su>",
@@ -25,9 +25,10 @@
         "@types/js-yaml": "^4.0.5",
         "cheerio": "1.0.0-rc.12",
         "csv-parse": "^5.5.0",
+        "eager-async-pool": "^1.0.0",
         "js-yaml": "4.1.0"
     },
     "typedoc": {
         "entryPoint": "index.d.ts"
     }
-}
+}

packages/tl/scripts/constants.ts

@@ -3,7 +3,7 @@ import * as url from 'url'
 
 export const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
 
-export const DOC_CACHE_FILE = join(__dirname, '.documentation.cache.json')
+export const DOC_CACHE_FILE = join(__dirname, '../data/documentation.cache.json')
 export const DESCRIPTIONS_YAML_FILE = join(__dirname, '../data/descriptions.yaml')
 export const API_SCHEMA_JSON_FILE = join(__dirname, '../api-schema.json')
 export const API_SCHEMA_DIFF_JSON_FILE = join(__dirname, '../diff.json')

packages/tl/scripts/documentation.ts

@@ -1,4 +1,5 @@
 import * as cheerio from 'cheerio'
+import { asyncPoolCallback } from 'eager-async-pool'
 import { readFile, writeFile } from 'fs/promises'
 import jsYaml from 'js-yaml'
 import { fileURLToPath } from 'node:url'
@@ -154,17 +155,16 @@ export async function fetchDocumentation(
 
     function log(str: string) {
         if (silent) return
-        while (str.length < prevSize) str += ' '
+        const oldPrevSize = prevSize
+        prevSize = str.length
+        while (str.length < oldPrevSize) str += ' '
 
         process.stdout.write('\r' + PROGRESS_CHARS[logPos] + ' ' + str)
 
-        prevSize = str.length
         logPos = (logPos + 1) % PROGRESS_CHARS.length
     }
 
-    for (const entry of schema.entries) {
+    async function fetchDocsForEntry(entry: TlEntry) {
-        log(`📥 ${entry.kind} ${entry.name}`)
-
         const url = `${domain}/${entry.kind === 'class' ? 'constructor' : 'method'}/${entry.name}`
 
         const html = await fetchRetry(url, {
@@ -173,7 +173,7 @@ export async function fetchDocumentation(
         const $ = cheerio.load(html)
         const content = $('#dev_page_content')
 
-        if (content.text().trim() === 'The page has not been saved') continue
+        if (content.text().trim() === 'The page has not been saved') return
 
         normalizeLinks(url, content)
 
@@ -236,7 +236,7 @@ export async function fetchDocumentation(
         ret[entry.kind === 'class' ? 'classes' : 'methods'][entry.name] = retClass
     }
 
-    for (const name in schema.unions) {
+    async function fetchDocsForUnion(name: string) {
         log(`📥 union ${name}`)
 
         const url = `${domain}/type/${name}`
@@ -247,7 +247,7 @@ export async function fetchDocumentation(
         const $ = cheerio.load(html)
         const content = $('#dev_page_content')
 
-        if (content.text().trim() === 'The page has not been saved') continue
+        if (content.text().trim() === 'The page has not been saved') return
 
         normalizeLinks(url, content)
 
@@ -255,6 +255,36 @@ export async function fetchDocumentation(
         if (description) ret.unions[name] = description
     }
 
+    await asyncPoolCallback(
+        fetchDocsForEntry,
+        schema.entries,
+        ({ item, error }) => {
+            if (error) {
+                console.log(`❌ ${item.kind} ${item.name} (${error.message})`)
+
+                return
+            }
+
+            log(`📥 ${item.kind} ${item.name}`)
+        },
+        { limit: 16 },
+    )
+
+    await asyncPoolCallback(
+        fetchDocsForUnion,
+        Object.keys(schema.unions),
+        ({ item, error }) => {
+            if (error) {
+                console.log(`❌ union ${item} (${error.message})`)
+
+                return
+            }
+
+            log(`📥 union ${item}`)
+        },
+        { limit: 16 },
+    )
+
     log('✨ Patching descriptions')
 
     const descriptionsYaml = jsYaml.load(await readFile(DESCRIPTIONS_YAML_FILE, 'utf8'))

packages/tl/scripts/gen-rsa-keys.ts

@@ -9,7 +9,7 @@ import { parsePublicKey } from '@mtcute/core/utils.js'
 import { TlPublicKey } from '../binary/rsa-keys.js'
 import { __dirname, ESM_PRELUDE } from './constants.js'
 
-const IN_TXT_FILE = join(__dirname, '.rsa-keys.txt')
+const IN_TXT_FILE = join(__dirname, '../data/rsa-keys.txt')
 const OUT_JS_FILE = join(__dirname, '../binary/rsa-keys.js')
 
 interface InputKey {

pnpm-lock.yaml

@@ -321,6 +321,9 @@ importers:
       csv-parse:
         specifier: ^5.5.0
         version: 5.5.0
+      eager-async-pool:
+        specifier: ^1.0.0
+        version: 1.0.0
       js-yaml:
         specifier: 4.1.0
         version: 4.1.0
@@ -2074,6 +2077,10 @@ packages:
       yargs: 17.7.2
     dev: true
 
+  /eager-async-pool@1.0.0:
+    resolution: {integrity: sha512-A2N+pbceYEz7O2KQ3TNQSSrZsivQ9i28cuNKRxfdT9QvRCoJ51pCP90hSNybOOBfoBHk15ZThJYrQWHo4h2UjA==}
+    dev: true
+
   /eastasianwidth@0.2.0:
     resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}