chore: use more utils from fuman

packages/core/src/utils/reloadable.ts

@@ -1,79 +0,0 @@
-import { timers } from '@fuman/utils'
-
-import { asyncResettable } from './function-utils.js'
-
-export interface ReloadableParams<Data> {
-    reload: (old?: Data) => Promise<Data>
-    getExpiresAt: (data: Data) => number
-    onError?: (err: unknown) => void
-    disableAutoReload?: boolean
-}
-
-export class Reloadable<Data> {
-    constructor(readonly params: ReloadableParams<Data>) {}
-
-    protected _data?: Data
-    protected _expiresAt = 0
-    protected _listeners: ((data: Data) => void)[] = []
-    protected _timeout?: timers.Timer
-
-    private _reload = asyncResettable(async () => {
-        const data = await this.params.reload(this._data)
-        this.setData(data)
-
-        this._listeners.forEach(cb => cb(data))
-    })
-
-    get isStale(): boolean {
-        return !this._data || this._expiresAt <= Date.now()
-    }
-
-    setData(data: Data): void {
-        const expiresAt = this.params.getExpiresAt(data)
-
-        this._data = data
-        this._expiresAt = expiresAt
-
-        if (this._timeout) timers.clearTimeout(this._timeout)
-
-        if (!this.params.disableAutoReload) {
-            this._timeout = timers.setTimeout(() => {
-                this._reload.reset()
-                this.update().catch((err: unknown) => {
-                    this.params.onError?.(err)
-                })
-            }, expiresAt - Date.now())
-        }
-    }
-
-    update(force = false): Promise<void> {
-        if (!force && !this.isStale) return Promise.resolve()
-
-        return this._reload.run()
-    }
-
-    onReload(cb: (data: Data) => void): void {
-        this._listeners.push(cb)
-    }
-
-    offReload(cb: (data: Data) => void): void {
-        const idx = this._listeners.indexOf(cb)
-        if (idx >= 0) this._listeners.splice(idx, 1)
-    }
-
-    getNow(): Data | undefined {
-        return this._data
-    }
-
-    async get(): Promise<Data> {
-        await this.update()
-
-        return this._data!
-    }
-
-    destroy(): void {
-        if (this._timeout) timers.clearTimeout(this._timeout)
-        this._listeners.length = 0
-        this._reload.reset()
-    }
-}

packages/tl/package.json

@@ -23,7 +23,8 @@
         "@mtcute/core": "workspace:^",
         "@mtcute/node": "workspace:^",
         "@mtcute/tl-utils": "workspace:^",
-        "@fuman/utils": "0.0.1",
+        "@fuman/utils": "0.0.1-fix.1",
+        "@fuman/fetch": "0.0.1-fix.2",
         "@types/js-yaml": "^4.0.5",
         "cheerio": "1.0.0-rc.12",
         "csv-parse": "^5.5.0",

packages/tl/scripts/documentation.ts

@@ -2,6 +2,7 @@ import { readFile, writeFile } from 'node:fs/promises'
 import { fileURLToPath } from 'node:url'
 import { createInterface } from 'node:readline'
 
+import { ffetchAddons, ffetchBase } from '@fuman/fetch'
 import * as cheerio from 'cheerio'
 import { asyncPool } from '@fuman/utils'
 import jsYaml from 'js-yaml'
@@ -29,7 +30,6 @@ import {
 import { applyDescriptionsYamlFile } from './process-descriptions-yaml.js'
 import type { TlPackedSchema } from './schema.js'
 import { packTlSchema, unpackTlSchema } from './schema.js'
-import { fetchRetry } from './utils.js'
 
 export interface CachedDocumentationEntry {
     comment?: string
@@ -45,6 +45,14 @@ export interface CachedDocumentation {
     unions: Record<string, string>
 }
 
+const ffetch = ffetchBase.extend({
+    addons: [
+        ffetchAddons.retry(),
+        ffetchAddons.timeout(),
+    ],
+    timeout: 30_000,
+})
+
 function normalizeLinks(url: string, el: cheerio.Cheerio<cheerio.Element>): void {
     el.find('a').each((i, _it) => {
         const it = cheerio.default(_it)
@@ -118,7 +126,7 @@ async function chooseDomainForDocs(headers: Record<string, string>): Promise<[nu
     let maxDomain = ''
 
     for (const domain of [CORE_DOMAIN, COREFORK_DOMAIN, BLOGFORK_DOMAIN]) {
-        const index = await fetchRetry(`${domain}/schema`, { headers })
+        const index = await ffetch(`${domain}/schema`, { headers }).text()
         const layerMatch = cheerio
             .load(index)('.dev_layer_select .dropdown-toggle')
             .text()
@@ -171,7 +179,7 @@ async function fetchAppConfigDocumentation() {
 
     const [, domain] = await chooseDomainForDocs(headers)
 
-    const page = await fetchRetry(`${domain}/api/config`, { headers })
+    const page = await ffetch(`${domain}/api/config`, { headers }).text()
     const $ = cheerio.load(page)
 
     const fields = $('p:icontains(typical fields included)').nextUntil('h3')
@@ -341,9 +349,7 @@ export async function fetchDocumentation(
     async function fetchDocsForEntry(entry: TlEntry) {
         const url = `${domain}/${entry.kind === 'class' ? 'constructor' : 'method'}/${entry.name}`
 
-        const html = await fetchRetry(url, {
+        const html = await ffetch(url, { headers }).text()
-            headers,
-        })
         const $ = cheerio.load(html)
         const content = $('#dev_page_content')
 
@@ -417,9 +423,7 @@ export async function fetchDocumentation(
 
         const url = `${domain}/type/${name}`
 
-        const html = await fetchRetry(url, {
+        const html = await ffetch(url, { headers }).text()
-            headers,
-        })
         const $ = cheerio.load(html)
         const content = $('#dev_page_content')
 

packages/tl/scripts/fetch-api.ts

@@ -23,6 +23,7 @@ import {
     writeTlEntryToString,
 } from '@mtcute/tl-utils'
 import { parseTlEntriesFromJson } from '@mtcute/tl-utils/json.js'
+import { ffetch } from '@fuman/fetch'
 
 import {
     API_SCHEMA_DIFF_JSON_FILE,
@@ -41,7 +42,6 @@ import {
 import { applyDocumentation, fetchDocumentation, getCachedDocumentation } from './documentation.js'
 import type { TlPackedSchema } from './schema.js'
 import { packTlSchema, unpackTlSchema } from './schema.js'
-import { fetchRetry } from './utils.js'
 
 const README_MD_FILE = join(__dirname, '../README.md')
 const PACKAGE_JSON_FILE = join(__dirname, '../package.json')
@@ -61,10 +61,8 @@ interface Schema {
 }
 
 async function fetchTdlibSchema(): Promise<Schema> {
-    const schema = await fetchRetry(TDLIB_SCHEMA)
+    const schema = await ffetch(TDLIB_SCHEMA).text()
-    const versionHtml = await fetch('https://raw.githubusercontent.com/tdlib/td/master/td/telegram/Version.h').then(
+    const versionHtml = await ffetch('https://raw.githubusercontent.com/tdlib/td/master/td/telegram/Version.h').text()
-        i => i.text(),
-    )
 
     const layer = versionHtml.match(/^constexpr int32 MTPROTO_LAYER = (\d+)/m)
     if (!layer) throw new Error('Layer number not available')
@@ -77,8 +75,8 @@ async function fetchTdlibSchema(): Promise<Schema> {
 }
 
 async function fetchTdesktopSchema(): Promise<Schema> {
-    const schema = await fetchRetry(TDESKTOP_SCHEMA)
+    const schema = await ffetch(TDESKTOP_SCHEMA).text()
-    const layerFile = await fetchRetry(TDESKTOP_LAYER)
+    const layerFile = await ffetch(TDESKTOP_LAYER).text()
     const layer = `${schema}\n\n${layerFile}`.match(/^\/\/ LAYER (\d+)/m)
     if (!layer) throw new Error('Layer number not available')
 
@@ -90,7 +88,7 @@ async function fetchTdesktopSchema(): Promise<Schema> {
 }
 
 async function fetchCoreSchema(domain = CORE_DOMAIN, name = 'Core'): Promise<Schema> {
-    const html = await fetchRetry(`${domain}/schema`)
+    const html = await ffetch(`${domain}/schema`).text()
     const $ = cheerio.load(html)
     // cheerio doesn't always unescape them
     const schema = $('.page_scheme code').text().replace(/&lt;/g, '<').replace(/&gt;/g, '>')
@@ -109,7 +107,7 @@ async function fetchCoreSchema(domain = CORE_DOMAIN, name = 'Core'): Promise<Sch
 }
 
 async function fetchWebkSchema(): Promise<Schema> {
-    const schema = await fetchRetry(WEBK_SCHEMA)
+    const schema = await ffetch(WEBK_SCHEMA).text()
     const json = JSON.parse(schema) as {
         layer: number
         API: object
@@ -133,7 +131,10 @@ async function fetchWebkSchema(): Promise<Schema> {
 }
 
 async function fetchWebaSchema(): Promise<Schema> {
-    const [schema, layerFile] = await Promise.all([fetchRetry(WEBA_SCHEMA), fetchRetry(WEBA_LAYER)])
+    const [schema, layerFile] = await Promise.all([
+        ffetch(WEBA_SCHEMA).text(),
+        ffetch(WEBA_LAYER).text(),
+    ])
 
     // const LAYER = 174;
     const version = layerFile.match(/^const LAYER = (\d+);$/m)

packages/tl/scripts/fetch-errors.ts

@@ -2,6 +2,7 @@ import { writeFile } from 'node:fs/promises'
 
 import { parse } from 'csv-parse/sync'
 import type { TlErrors } from '@mtcute/tl-utils'
+import { ffetch } from '@fuman/fetch'
 
 import { ERRORS_JSON_FILE } from './constants.js'
 
@@ -27,11 +28,11 @@ interface TelegramErrorsSpec {
 }
 
 async function fetchFromTelegram(errors: TlErrors) {
-    const page = await fetch(ERRORS_PAGE_TG).then(it => it.text())
+    const page = await ffetch(ERRORS_PAGE_TG).text()
     const jsonUrl = page.match(/can be found <a href="([^"]+)">here »<\/a>/i)?.[1]
     if (!jsonUrl) throw new Error('Cannot find JSON URL')
 
-    const json = (await fetch(new URL(jsonUrl, ERRORS_PAGE_TG)).then(it => it.json())) as TelegramErrorsSpec
+    const json = await ffetch(new URL(jsonUrl, ERRORS_PAGE_TG).href).json<TelegramErrorsSpec>()
 
     // since nobody fucking guarantees that .descriptions
     // will have description for each described here (or vice versa),
@@ -122,13 +123,9 @@ async function fetchFromTelegram(errors: TlErrors) {
 }
 
 async function fetchFromTelethon(errors: TlErrors) {
-    const csv = await fetch(ERRORS_PAGE_TELETHON)
+    const csv = await ffetch(ERRORS_PAGE_TELETHON).text()
 
-    if (!csv.body) {
+    const records = parse(csv, {
-        throw new Error('No body in response')
-    }
-
-    const records = parse(await csv.text(), {
         columns: true,
         skip_empty_lines: true,
     }) as {

packages/tl/scripts/fetch-mtp.ts

@@ -4,12 +4,12 @@ import { writeFile } from 'node:fs/promises'
 
 import * as cheerio from 'cheerio'
 import { parseTlToEntries } from '@mtcute/tl-utils'
+import { ffetch } from '@fuman/fetch'
 
 import { CORE_DOMAIN, MTP_SCHEMA_JSON_FILE } from './constants.js'
-import { fetchRetry } from './utils.js'
 
 async function fetchMtprotoSchema(): Promise<string> {
-    const html = await fetchRetry(`${CORE_DOMAIN}/schema/mtproto`)
+    const html = await ffetch(`${CORE_DOMAIN}/schema/mtproto`).text()
     const $ = cheerio.load(html)
 
     // cheerio doesn't always unescape them

packages/tl/scripts/utils.ts

@@ -1,11 +0,0 @@
-export async function fetchRetry(url: string, params?: RequestInit, retries = 5): Promise<string> {
-    while (true) {
-        try {
-            return await fetch(url, params).then(i => i.text())
-        } catch (e) {
-            if (!retries--) {
-                throw e
-            }
-        }
-    }
-}