From 7adcc2b95c607eaad58b4d394637ae102b3a68d1 Mon Sep 17 00:00:00 2001 From: alina sireneva Date: Tue, 19 Nov 2024 18:35:02 +0300 Subject: [PATCH] chore: use more utils from fuman --- packages/core/src/utils/reloadable.ts | 79 --------------------------- packages/tl/package.json | 3 +- packages/tl/scripts/documentation.ts | 22 +++++--- packages/tl/scripts/fetch-api.ts | 21 +++---- packages/tl/scripts/fetch-errors.ts | 13 ++--- packages/tl/scripts/fetch-mtp.ts | 4 +- packages/tl/scripts/utils.ts | 11 ---- 7 files changed, 33 insertions(+), 120 deletions(-) delete mode 100644 packages/core/src/utils/reloadable.ts delete mode 100644 packages/tl/scripts/utils.ts diff --git a/packages/core/src/utils/reloadable.ts b/packages/core/src/utils/reloadable.ts deleted file mode 100644 index d0d806e9..00000000 --- a/packages/core/src/utils/reloadable.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { timers } from '@fuman/utils' - -import { asyncResettable } from './function-utils.js' - -export interface ReloadableParams { - reload: (old?: Data) => Promise - getExpiresAt: (data: Data) => number - onError?: (err: unknown) => void - disableAutoReload?: boolean -} - -export class Reloadable { - constructor(readonly params: ReloadableParams) {} - - protected _data?: Data - protected _expiresAt = 0 - protected _listeners: ((data: Data) => void)[] = [] - protected _timeout?: timers.Timer - - private _reload = asyncResettable(async () => { - const data = await this.params.reload(this._data) - this.setData(data) - - this._listeners.forEach(cb => cb(data)) - }) - - get isStale(): boolean { - return !this._data || this._expiresAt <= Date.now() - } - - setData(data: Data): void { - const expiresAt = this.params.getExpiresAt(data) - - this._data = data - this._expiresAt = expiresAt - - if (this._timeout) timers.clearTimeout(this._timeout) - - if (!this.params.disableAutoReload) { - this._timeout = timers.setTimeout(() => { - this._reload.reset() - this.update().catch((err: unknown) => { - this.params.onError?.(err) - }) - }, expiresAt - Date.now()) - } - } - - update(force = false): Promise { - if (!force && !this.isStale) return Promise.resolve() - - return this._reload.run() - } - - onReload(cb: (data: Data) => void): void { - this._listeners.push(cb) - } - - offReload(cb: (data: Data) => void): void { - const idx = this._listeners.indexOf(cb) - if (idx >= 0) this._listeners.splice(idx, 1) - } - - getNow(): Data | undefined { - return this._data - } - - async get(): Promise { - await this.update() - - return this._data! - } - - destroy(): void { - if (this._timeout) timers.clearTimeout(this._timeout) - this._listeners.length = 0 - this._reload.reset() - } -} diff --git a/packages/tl/package.json b/packages/tl/package.json index 1aa33bb6..3249649e 100644 --- a/packages/tl/package.json +++ b/packages/tl/package.json @@ -23,7 +23,8 @@ "@mtcute/core": "workspace:^", "@mtcute/node": "workspace:^", "@mtcute/tl-utils": "workspace:^", - "@fuman/utils": "0.0.1", + "@fuman/utils": "0.0.1-fix.1", + "@fuman/fetch": "0.0.1-fix.2", "@types/js-yaml": "^4.0.5", "cheerio": "1.0.0-rc.12", "csv-parse": "^5.5.0", diff --git a/packages/tl/scripts/documentation.ts b/packages/tl/scripts/documentation.ts index a40e8dd3..d5eb76aa 100644 --- a/packages/tl/scripts/documentation.ts +++ b/packages/tl/scripts/documentation.ts @@ -2,6 +2,7 @@ import { readFile, writeFile } from 'node:fs/promises' import { fileURLToPath } from 'node:url' import { createInterface } from 'node:readline' +import { ffetchAddons, ffetchBase } from '@fuman/fetch' import * as cheerio from 'cheerio' import { asyncPool } from '@fuman/utils' import jsYaml from 'js-yaml' @@ -29,7 +30,6 @@ import { import { applyDescriptionsYamlFile } from './process-descriptions-yaml.js' import type { TlPackedSchema } from './schema.js' import { packTlSchema, unpackTlSchema } from './schema.js' -import { fetchRetry } from './utils.js' export interface CachedDocumentationEntry { comment?: string @@ -45,6 +45,14 @@ export interface CachedDocumentation { unions: Record } +const ffetch = ffetchBase.extend({ + addons: [ + ffetchAddons.retry(), + ffetchAddons.timeout(), + ], + timeout: 30_000, +}) + function normalizeLinks(url: string, el: cheerio.Cheerio): void { el.find('a').each((i, _it) => { const it = cheerio.default(_it) @@ -118,7 +126,7 @@ async function chooseDomainForDocs(headers: Record): Promise<[nu let maxDomain = '' for (const domain of [CORE_DOMAIN, COREFORK_DOMAIN, BLOGFORK_DOMAIN]) { - const index = await fetchRetry(`${domain}/schema`, { headers }) + const index = await ffetch(`${domain}/schema`, { headers }).text() const layerMatch = cheerio .load(index)('.dev_layer_select .dropdown-toggle') .text() @@ -171,7 +179,7 @@ async function fetchAppConfigDocumentation() { const [, domain] = await chooseDomainForDocs(headers) - const page = await fetchRetry(`${domain}/api/config`, { headers }) + const page = await ffetch(`${domain}/api/config`, { headers }).text() const $ = cheerio.load(page) const fields = $('p:icontains(typical fields included)').nextUntil('h3') @@ -341,9 +349,7 @@ export async function fetchDocumentation( async function fetchDocsForEntry(entry: TlEntry) { const url = `${domain}/${entry.kind === 'class' ? 'constructor' : 'method'}/${entry.name}` - const html = await fetchRetry(url, { - headers, - }) + const html = await ffetch(url, { headers }).text() const $ = cheerio.load(html) const content = $('#dev_page_content') @@ -417,9 +423,7 @@ export async function fetchDocumentation( const url = `${domain}/type/${name}` - const html = await fetchRetry(url, { - headers, - }) + const html = await ffetch(url, { headers }).text() const $ = cheerio.load(html) const content = $('#dev_page_content') diff --git a/packages/tl/scripts/fetch-api.ts b/packages/tl/scripts/fetch-api.ts index 15016c36..61254ac2 100644 --- a/packages/tl/scripts/fetch-api.ts +++ b/packages/tl/scripts/fetch-api.ts @@ -23,6 +23,7 @@ import { writeTlEntryToString, } from '@mtcute/tl-utils' import { parseTlEntriesFromJson } from '@mtcute/tl-utils/json.js' +import { ffetch } from '@fuman/fetch' import { API_SCHEMA_DIFF_JSON_FILE, @@ -41,7 +42,6 @@ import { import { applyDocumentation, fetchDocumentation, getCachedDocumentation } from './documentation.js' import type { TlPackedSchema } from './schema.js' import { packTlSchema, unpackTlSchema } from './schema.js' -import { fetchRetry } from './utils.js' const README_MD_FILE = join(__dirname, '../README.md') const PACKAGE_JSON_FILE = join(__dirname, '../package.json') @@ -61,10 +61,8 @@ interface Schema { } async function fetchTdlibSchema(): Promise { - const schema = await fetchRetry(TDLIB_SCHEMA) - const versionHtml = await fetch('https://raw.githubusercontent.com/tdlib/td/master/td/telegram/Version.h').then( - i => i.text(), - ) + const schema = await ffetch(TDLIB_SCHEMA).text() + const versionHtml = await ffetch('https://raw.githubusercontent.com/tdlib/td/master/td/telegram/Version.h').text() const layer = versionHtml.match(/^constexpr int32 MTPROTO_LAYER = (\d+)/m) if (!layer) throw new Error('Layer number not available') @@ -77,8 +75,8 @@ async function fetchTdlibSchema(): Promise { } async function fetchTdesktopSchema(): Promise { - const schema = await fetchRetry(TDESKTOP_SCHEMA) - const layerFile = await fetchRetry(TDESKTOP_LAYER) + const schema = await ffetch(TDESKTOP_SCHEMA).text() + const layerFile = await ffetch(TDESKTOP_LAYER).text() const layer = `${schema}\n\n${layerFile}`.match(/^\/\/ LAYER (\d+)/m) if (!layer) throw new Error('Layer number not available') @@ -90,7 +88,7 @@ async function fetchTdesktopSchema(): Promise { } async function fetchCoreSchema(domain = CORE_DOMAIN, name = 'Core'): Promise { - const html = await fetchRetry(`${domain}/schema`) + const html = await ffetch(`${domain}/schema`).text() const $ = cheerio.load(html) // cheerio doesn't always unescape them const schema = $('.page_scheme code').text().replace(/</g, '<').replace(/>/g, '>') @@ -109,7 +107,7 @@ async function fetchCoreSchema(domain = CORE_DOMAIN, name = 'Core'): Promise { - const schema = await fetchRetry(WEBK_SCHEMA) + const schema = await ffetch(WEBK_SCHEMA).text() const json = JSON.parse(schema) as { layer: number API: object @@ -133,7 +131,10 @@ async function fetchWebkSchema(): Promise { } async function fetchWebaSchema(): Promise { - const [schema, layerFile] = await Promise.all([fetchRetry(WEBA_SCHEMA), fetchRetry(WEBA_LAYER)]) + const [schema, layerFile] = await Promise.all([ + ffetch(WEBA_SCHEMA).text(), + ffetch(WEBA_LAYER).text(), + ]) // const LAYER = 174; const version = layerFile.match(/^const LAYER = (\d+);$/m) diff --git a/packages/tl/scripts/fetch-errors.ts b/packages/tl/scripts/fetch-errors.ts index 3a391958..f56726f8 100644 --- a/packages/tl/scripts/fetch-errors.ts +++ b/packages/tl/scripts/fetch-errors.ts @@ -2,6 +2,7 @@ import { writeFile } from 'node:fs/promises' import { parse } from 'csv-parse/sync' import type { TlErrors } from '@mtcute/tl-utils' +import { ffetch } from '@fuman/fetch' import { ERRORS_JSON_FILE } from './constants.js' @@ -27,11 +28,11 @@ interface TelegramErrorsSpec { } async function fetchFromTelegram(errors: TlErrors) { - const page = await fetch(ERRORS_PAGE_TG).then(it => it.text()) + const page = await ffetch(ERRORS_PAGE_TG).text() const jsonUrl = page.match(/can be found here ยป<\/a>/i)?.[1] if (!jsonUrl) throw new Error('Cannot find JSON URL') - const json = (await fetch(new URL(jsonUrl, ERRORS_PAGE_TG)).then(it => it.json())) as TelegramErrorsSpec + const json = await ffetch(new URL(jsonUrl, ERRORS_PAGE_TG).href).json() // since nobody fucking guarantees that .descriptions // will have description for each described here (or vice versa), @@ -122,13 +123,9 @@ async function fetchFromTelegram(errors: TlErrors) { } async function fetchFromTelethon(errors: TlErrors) { - const csv = await fetch(ERRORS_PAGE_TELETHON) + const csv = await ffetch(ERRORS_PAGE_TELETHON).text() - if (!csv.body) { - throw new Error('No body in response') - } - - const records = parse(await csv.text(), { + const records = parse(csv, { columns: true, skip_empty_lines: true, }) as { diff --git a/packages/tl/scripts/fetch-mtp.ts b/packages/tl/scripts/fetch-mtp.ts index 3badea18..bdae20b9 100644 --- a/packages/tl/scripts/fetch-mtp.ts +++ b/packages/tl/scripts/fetch-mtp.ts @@ -4,12 +4,12 @@ import { writeFile } from 'node:fs/promises' import * as cheerio from 'cheerio' import { parseTlToEntries } from '@mtcute/tl-utils' +import { ffetch } from '@fuman/fetch' import { CORE_DOMAIN, MTP_SCHEMA_JSON_FILE } from './constants.js' -import { fetchRetry } from './utils.js' async function fetchMtprotoSchema(): Promise { - const html = await fetchRetry(`${CORE_DOMAIN}/schema/mtproto`) + const html = await ffetch(`${CORE_DOMAIN}/schema/mtproto`).text() const $ = cheerio.load(html) // cheerio doesn't always unescape them diff --git a/packages/tl/scripts/utils.ts b/packages/tl/scripts/utils.ts deleted file mode 100644 index a26770a1..00000000 --- a/packages/tl/scripts/utils.ts +++ /dev/null @@ -1,11 +0,0 @@ -export async function fetchRetry(url: string, params?: RequestInit, retries = 5): Promise { - while (true) { - try { - return await fetch(url, params).then(i => i.text()) - } catch (e) { - if (!retries--) { - throw e - } - } - } -}