scripts/utils/fetch.ts

import { createWriteStream } from 'node:fs'

import { type FfetchAddon, ffetchAddons, ffetchBase, type FfetchResultInternals } from '@fuman/fetch'
import { toughCookieAddon } from '@fuman/fetch/tough'
import { ffetchZodAdapter } from '@fuman/fetch/zod'
import { webReadableToFuman, write } from '@fuman/io'
import { nodeWritableToFuman } from '@fuman/node'
import { type CheerioAPI, load } from 'cheerio'
import { ProxyAgent } from 'undici'

const cheerioAddon: FfetchAddon<object, { cheerio: () => Promise<CheerioAPI> }> = {
  response: {
    async cheerio(this: FfetchResultInternals<object>) {
      this._headers ??= {}
      this._headers.Accept ??= 'text/html; charset=utf-8'
      return load(await this.text())
    },
  },
}

export const ffetch = ffetchBase.extend({
  addons: [
    ffetchAddons.parser(ffetchZodAdapter()),
    cheerioAddon,
    toughCookieAddon(),
  ],
  extra: {
    dispatcher: process.env.http_proxy ? new ProxyAgent(process.env.http_proxy) : undefined,
  } as any,
})

export async function downloadStream(stream: ReadableStream, path: string) {
  const file = nodeWritableToFuman(createWriteStream(path))
  await write.pipe(file, webReadableToFuman(stream))
  file.close()
}

export async function downloadFile(url: string, path: string, extra?: Parameters<typeof ffetch>[1]) {
  const stream = await ffetch(url, extra).stream()
  await downloadStream(stream, path)
}