41 lines
1.4 KiB
TypeScript
41 lines
1.4 KiB
TypeScript
import { createWriteStream } from 'node:fs'
|
|
|
|
import { type FfetchAddon, ffetchAddons, ffetchBase, type FfetchResultInternals } from '@fuman/fetch'
|
|
import { toughCookieAddon } from '@fuman/fetch/tough'
|
|
import { ffetchZodAdapter } from '@fuman/fetch/zod'
|
|
import { webReadableToFuman, write } from '@fuman/io'
|
|
import { nodeWritableToFuman } from '@fuman/node'
|
|
import { type CheerioAPI, load } from 'cheerio'
|
|
import { ProxyAgent } from 'undici'
|
|
|
|
const cheerioAddon: FfetchAddon<object, { cheerio: () => Promise<CheerioAPI> }> = {
|
|
response: {
|
|
async cheerio(this: FfetchResultInternals<object>) {
|
|
this._headers ??= {}
|
|
this._headers.Accept ??= 'text/html; charset=utf-8'
|
|
return load(await this.text())
|
|
},
|
|
},
|
|
}
|
|
|
|
export const ffetch = ffetchBase.extend({
|
|
addons: [
|
|
ffetchAddons.parser(ffetchZodAdapter()),
|
|
cheerioAddon,
|
|
toughCookieAddon(),
|
|
],
|
|
extra: {
|
|
dispatcher: process.env.http_proxy ? new ProxyAgent(process.env.http_proxy) : undefined,
|
|
} as any,
|
|
})
|
|
|
|
export async function downloadStream(stream: ReadableStream, path: string) {
|
|
const file = nodeWritableToFuman(createWriteStream(path))
|
|
await write.pipe(file, webReadableToFuman(stream))
|
|
file.close()
|
|
}
|
|
|
|
export async function downloadFile(url: string, path: string, extra?: Parameters<typeof ffetch>[1]) {
|
|
const stream = await ffetch(url, extra).stream()
|
|
await downloadStream(stream, path)
|
|
}
|