fix: improved downloadToFile API
This commit is contained in:
parent
9791f8faae
commit
04c702dfd2
7 changed files with 43 additions and 38 deletions
|
@ -267,6 +267,7 @@ import {
|
||||||
DeleteMessageUpdate,
|
DeleteMessageUpdate,
|
||||||
DeleteStoryUpdate,
|
DeleteStoryUpdate,
|
||||||
Dialog,
|
Dialog,
|
||||||
|
FileDownloadLocation,
|
||||||
FileDownloadParameters,
|
FileDownloadParameters,
|
||||||
FormattedString,
|
FormattedString,
|
||||||
ForumTopic,
|
ForumTopic,
|
||||||
|
@ -2125,7 +2126,7 @@ export interface TelegramClient extends BaseTelegramClient {
|
||||||
*
|
*
|
||||||
* @param params File download parameters
|
* @param params File download parameters
|
||||||
*/
|
*/
|
||||||
downloadAsBuffer(params: FileDownloadParameters): Promise<Uint8Array>
|
downloadAsBuffer(location: FileDownloadLocation, params?: FileDownloadParameters): Promise<Uint8Array>
|
||||||
/**
|
/**
|
||||||
* Download a remote file to a local file (only for NodeJS).
|
* Download a remote file to a local file (only for NodeJS).
|
||||||
* Promise will resolve once the download is complete.
|
* Promise will resolve once the download is complete.
|
||||||
|
@ -2135,7 +2136,7 @@ export interface TelegramClient extends BaseTelegramClient {
|
||||||
* @param filename Local file name to which the remote file will be downloaded
|
* @param filename Local file name to which the remote file will be downloaded
|
||||||
* @param params File download parameters
|
* @param params File download parameters
|
||||||
*/
|
*/
|
||||||
downloadToFile(filename: string, params: FileDownloadParameters): Promise<void>
|
downloadToFile(filename: string, location: FileDownloadLocation, params?: FileDownloadParameters): Promise<void>
|
||||||
/**
|
/**
|
||||||
* Download a file and return it as an iterable, which yields file contents
|
* Download a file and return it as an iterable, which yields file contents
|
||||||
* in chunks of a given size. Order of the chunks is guaranteed to be
|
* in chunks of a given size. Order of the chunks is guaranteed to be
|
||||||
|
@ -2145,7 +2146,7 @@ export interface TelegramClient extends BaseTelegramClient {
|
||||||
*
|
*
|
||||||
* @param params Download parameters
|
* @param params Download parameters
|
||||||
*/
|
*/
|
||||||
downloadAsIterable(params: FileDownloadParameters): AsyncIterableIterator<Uint8Array>
|
downloadAsIterable(input: FileDownloadLocation, params?: FileDownloadParameters): AsyncIterableIterator<Uint8Array>
|
||||||
/**
|
/**
|
||||||
* Download a file and return it as a readable stream,
|
* Download a file and return it as a readable stream,
|
||||||
* streaming file contents.
|
* streaming file contents.
|
||||||
|
@ -2154,7 +2155,7 @@ export interface TelegramClient extends BaseTelegramClient {
|
||||||
*
|
*
|
||||||
* @param params File download parameters
|
* @param params File download parameters
|
||||||
*/
|
*/
|
||||||
downloadAsStream(params: FileDownloadParameters): ReadableStream<Uint8Array>
|
downloadAsStream(location: FileDownloadLocation, params?: FileDownloadParameters): ReadableStream<Uint8Array>
|
||||||
/**
|
/**
|
||||||
* Normalize a {@link InputFileLike} to `InputFile`,
|
* Normalize a {@link InputFileLike} to `InputFile`,
|
||||||
* uploading it if needed.
|
* uploading it if needed.
|
||||||
|
|
|
@ -36,6 +36,7 @@ import {
|
||||||
DeleteMessageUpdate,
|
DeleteMessageUpdate,
|
||||||
DeleteStoryUpdate,
|
DeleteStoryUpdate,
|
||||||
Dialog,
|
Dialog,
|
||||||
|
FileDownloadLocation,
|
||||||
FileDownloadParameters,
|
FileDownloadParameters,
|
||||||
FormattedString,
|
FormattedString,
|
||||||
ForumTopic,
|
ForumTopic,
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import { BaseTelegramClient } from '@mtcute/core'
|
import { BaseTelegramClient } from '@mtcute/core'
|
||||||
import { concatBuffers } from '@mtcute/core/utils.js'
|
import { concatBuffers } from '@mtcute/core/utils.js'
|
||||||
|
|
||||||
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
|
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
|
||||||
import { downloadAsIterable } from './download-iterable.js'
|
import { downloadAsIterable } from './download-iterable.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -14,15 +14,16 @@ import { downloadAsIterable } from './download-iterable.js'
|
||||||
*/
|
*/
|
||||||
export async function downloadAsBuffer(
|
export async function downloadAsBuffer(
|
||||||
client: BaseTelegramClient,
|
client: BaseTelegramClient,
|
||||||
params: FileDownloadParameters,
|
location: FileDownloadLocation,
|
||||||
|
params?: FileDownloadParameters,
|
||||||
): Promise<Uint8Array> {
|
): Promise<Uint8Array> {
|
||||||
if (params.location instanceof FileLocation && ArrayBuffer.isView(params.location.location)) {
|
if (location instanceof FileLocation && ArrayBuffer.isView(location.location)) {
|
||||||
return params.location.location
|
return location.location
|
||||||
}
|
}
|
||||||
|
|
||||||
const chunks = []
|
const chunks = []
|
||||||
|
|
||||||
for await (const chunk of downloadAsIterable(client, params)) {
|
for await (const chunk of downloadAsIterable(client, location, params)) {
|
||||||
chunks.push(chunk)
|
chunks.push(chunk)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,7 @@ import { createRequire } from 'module'
|
||||||
|
|
||||||
import { BaseTelegramClient, MtUnsupportedError } from '@mtcute/core'
|
import { BaseTelegramClient, MtUnsupportedError } from '@mtcute/core'
|
||||||
|
|
||||||
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
|
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
|
||||||
import { downloadAsIterable } from './download-iterable.js'
|
import { downloadAsIterable } from './download-iterable.js'
|
||||||
|
|
||||||
let fs: typeof import('fs') | null = null
|
let fs: typeof import('fs') | null = null
|
||||||
|
@ -24,15 +24,16 @@ try {
|
||||||
export async function downloadToFile(
|
export async function downloadToFile(
|
||||||
client: BaseTelegramClient,
|
client: BaseTelegramClient,
|
||||||
filename: string,
|
filename: string,
|
||||||
params: FileDownloadParameters,
|
location: FileDownloadLocation,
|
||||||
|
params?: FileDownloadParameters,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!fs) {
|
if (!fs) {
|
||||||
throw new MtUnsupportedError('Downloading to file is only supported in NodeJS')
|
throw new MtUnsupportedError('Downloading to file is only supported in NodeJS')
|
||||||
}
|
}
|
||||||
|
|
||||||
if (params.location instanceof FileLocation && ArrayBuffer.isView(params.location.location)) {
|
if (location instanceof FileLocation && ArrayBuffer.isView(location.location)) {
|
||||||
// early return for inline files
|
// early return for inline files
|
||||||
const buf = params.location.location
|
const buf = location.location
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
fs!.writeFile(filename, buf, (err) => {
|
fs!.writeFile(filename, buf, (err) => {
|
||||||
|
@ -44,7 +45,7 @@ export async function downloadToFile(
|
||||||
|
|
||||||
const output = fs.createWriteStream(filename)
|
const output = fs.createWriteStream(filename)
|
||||||
|
|
||||||
if (params.abortSignal) {
|
if (params?.abortSignal) {
|
||||||
params.abortSignal.addEventListener('abort', () => {
|
params.abortSignal.addEventListener('abort', () => {
|
||||||
client.log.debug('aborting file download %s - cleaning up', filename)
|
client.log.debug('aborting file download %s - cleaning up', filename)
|
||||||
output.destroy()
|
output.destroy()
|
||||||
|
@ -52,7 +53,7 @@ export async function downloadToFile(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
for await (const chunk of downloadAsIterable(client, params)) {
|
for await (const chunk of downloadAsIterable(client, location, params)) {
|
||||||
output.write(chunk)
|
output.write(chunk)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,7 @@ import { BaseTelegramClient, ConnectionKind, MtArgumentError, MtUnsupportedError
|
||||||
import { ConditionVariable } from '@mtcute/core/utils.js'
|
import { ConditionVariable } from '@mtcute/core/utils.js'
|
||||||
import { fileIdToInputFileLocation, fileIdToInputWebFileLocation, parseFileId } from '@mtcute/file-id'
|
import { fileIdToInputFileLocation, fileIdToInputWebFileLocation, parseFileId } from '@mtcute/file-id'
|
||||||
|
|
||||||
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
|
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
|
||||||
import { determinePartSize } from '../../utils/file-utils.js'
|
import { determinePartSize } from '../../utils/file-utils.js'
|
||||||
|
|
||||||
// small files (less than 128 kb) are downloaded using the "downloadSmall" pool
|
// small files (less than 128 kb) are downloaded using the "downloadSmall" pool
|
||||||
|
@ -20,18 +20,18 @@ const REQUESTS_PER_CONNECTION = 3 // some arbitrary magic value that seems to wo
|
||||||
*/
|
*/
|
||||||
export async function* downloadAsIterable(
|
export async function* downloadAsIterable(
|
||||||
client: BaseTelegramClient,
|
client: BaseTelegramClient,
|
||||||
params: FileDownloadParameters,
|
input: FileDownloadLocation,
|
||||||
|
params?: FileDownloadParameters,
|
||||||
): AsyncIterableIterator<Uint8Array> {
|
): AsyncIterableIterator<Uint8Array> {
|
||||||
const offset = params.offset ?? 0
|
const offset = params?.offset ?? 0
|
||||||
|
|
||||||
if (offset % 4096 !== 0) {
|
if (offset % 4096 !== 0) {
|
||||||
throw new MtArgumentError(`Invalid offset: ${offset}. Must be divisible by 4096`)
|
throw new MtArgumentError(`Invalid offset: ${offset}. Must be divisible by 4096`)
|
||||||
}
|
}
|
||||||
|
|
||||||
let dcId = params.dcId
|
let dcId = params?.dcId
|
||||||
let fileSize = params.fileSize
|
let fileSize = params?.fileSize
|
||||||
|
|
||||||
const input = params.location
|
|
||||||
let location: tl.TypeInputFileLocation | tl.TypeInputWebFileLocation
|
let location: tl.TypeInputFileLocation | tl.TypeInputWebFileLocation
|
||||||
if (input instanceof FileLocation) {
|
if (input instanceof FileLocation) {
|
||||||
let locationInner = input.location
|
let locationInner = input.location
|
||||||
|
@ -63,7 +63,7 @@ export async function* downloadAsIterable(
|
||||||
// we will receive a FileMigrateError in case this is invalid
|
// we will receive a FileMigrateError in case this is invalid
|
||||||
if (!dcId) dcId = client.network.getPrimaryDcId()
|
if (!dcId) dcId = client.network.getPrimaryDcId()
|
||||||
|
|
||||||
const partSizeKb = params.partSize ?? (fileSize ? determinePartSize(fileSize) : 64)
|
const partSizeKb = params?.partSize ?? (fileSize ? determinePartSize(fileSize) : 64)
|
||||||
|
|
||||||
if (partSizeKb % 4 !== 0) {
|
if (partSizeKb % 4 !== 0) {
|
||||||
throw new MtArgumentError(`Invalid part size: ${partSizeKb}. Must be divisible by 4.`)
|
throw new MtArgumentError(`Invalid part size: ${partSizeKb}. Must be divisible by 4.`)
|
||||||
|
@ -71,7 +71,7 @@ export async function* downloadAsIterable(
|
||||||
|
|
||||||
const chunkSize = partSizeKb * 1024
|
const chunkSize = partSizeKb * 1024
|
||||||
|
|
||||||
let limitBytes = params.limit ?? fileSize ?? Infinity
|
let limitBytes = params?.limit ?? fileSize ?? Infinity
|
||||||
if (limitBytes === 0) return
|
if (limitBytes === 0) return
|
||||||
|
|
||||||
let numChunks = limitBytes === Infinity ? Infinity : ~~((limitBytes + chunkSize - offset - 1) / chunkSize)
|
let numChunks = limitBytes === Infinity ? Infinity : ~~((limitBytes + chunkSize - offset - 1) / chunkSize)
|
||||||
|
@ -111,7 +111,7 @@ export async function* downloadAsIterable(
|
||||||
offset: chunkSize * chunk,
|
offset: chunkSize * chunk,
|
||||||
limit: chunkSize,
|
limit: chunkSize,
|
||||||
},
|
},
|
||||||
{ dcId, kind: connectionKind, abortSignal: params.abortSignal },
|
{ dcId, kind: connectionKind, abortSignal: params?.abortSignal },
|
||||||
)
|
)
|
||||||
} catch (e: unknown) {
|
} catch (e: unknown) {
|
||||||
if (!tl.RpcError.is(e)) throw e
|
if (!tl.RpcError.is(e)) throw e
|
||||||
|
@ -175,7 +175,7 @@ export async function* downloadAsIterable(
|
||||||
|
|
||||||
position += buf.length
|
position += buf.length
|
||||||
|
|
||||||
params.progressCallback?.(position, limitBytes)
|
params?.progressCallback?.(position, limitBytes)
|
||||||
|
|
||||||
yield buf
|
yield buf
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import { BaseTelegramClient } from '@mtcute/core'
|
import { BaseTelegramClient } from '@mtcute/core'
|
||||||
|
|
||||||
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
|
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
|
||||||
import { bufferToStream } from '../../utils/stream-utils.js'
|
import { bufferToStream } from '../../utils/stream-utils.js'
|
||||||
import { downloadAsIterable } from './download-iterable.js'
|
import { downloadAsIterable } from './download-iterable.js'
|
||||||
|
|
||||||
|
@ -12,16 +12,17 @@ import { downloadAsIterable } from './download-iterable.js'
|
||||||
*/
|
*/
|
||||||
export function downloadAsStream(
|
export function downloadAsStream(
|
||||||
client: BaseTelegramClient,
|
client: BaseTelegramClient,
|
||||||
params: FileDownloadParameters,
|
location: FileDownloadLocation,
|
||||||
|
params?: FileDownloadParameters,
|
||||||
): ReadableStream<Uint8Array> {
|
): ReadableStream<Uint8Array> {
|
||||||
if (params.location instanceof FileLocation && ArrayBuffer.isView(params.location.location)) {
|
if (location instanceof FileLocation && ArrayBuffer.isView(location.location)) {
|
||||||
return bufferToStream(params.location.location)
|
return bufferToStream(location.location)
|
||||||
}
|
}
|
||||||
|
|
||||||
const cancel = new AbortController()
|
const cancel = new AbortController()
|
||||||
|
|
||||||
if (params.abortSignal) {
|
if (params?.abortSignal) {
|
||||||
params.abortSignal.addEventListener('abort', () => {
|
params?.abortSignal.addEventListener('abort', () => {
|
||||||
cancel.abort()
|
cancel.abort()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -29,7 +30,7 @@ export function downloadAsStream(
|
||||||
return new ReadableStream<Uint8Array>({
|
return new ReadableStream<Uint8Array>({
|
||||||
start(controller) {
|
start(controller) {
|
||||||
(async () => {
|
(async () => {
|
||||||
for await (const chunk of downloadAsIterable(client, params)) {
|
for await (const chunk of downloadAsIterable(client, location, params)) {
|
||||||
controller.enqueue(chunk)
|
controller.enqueue(chunk)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,13 +51,13 @@ export type InputFileLike =
|
||||||
| tl.TypeInputMedia
|
| tl.TypeInputMedia
|
||||||
| tdFileId.RawFullRemoteFileLocation
|
| tdFileId.RawFullRemoteFileLocation
|
||||||
|
|
||||||
export interface FileDownloadParameters {
|
/**
|
||||||
/**
|
* File location which should be downloaded.
|
||||||
* File location which should be downloaded.
|
* You can also provide TDLib and Bot API compatible File ID
|
||||||
* You can also provide TDLib and Bot API compatible File ID
|
*/
|
||||||
*/
|
export type FileDownloadLocation = tl.TypeInputFileLocation | tl.TypeInputWebFileLocation | FileLocation | string
|
||||||
location: tl.TypeInputFileLocation | tl.TypeInputWebFileLocation | FileLocation | string
|
|
||||||
|
|
||||||
|
export interface FileDownloadParameters {
|
||||||
/**
|
/**
|
||||||
* Total file size, if known.
|
* Total file size, if known.
|
||||||
* Used to determine upload part size.
|
* Used to determine upload part size.
|
||||||
|
|
Loading…
Reference in a new issue