teidesu/mtcute
1
0
Fork 0
Code Issues 15 Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: improved downloadToFile API

Browse source
This commit is contained in:
alina 🌸 2023-10-27 19:44:40 +03:00
parent 9791f8faae
commit 04c702dfd2
Signed by: teidesu
SSH key fingerprint: SHA256:uNeCpw6aTSU4aIObXLvHfLkDa82HWH9EiOj9AXOIRpI
7 changed files with 43 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
packages/client/src/client.ts
View file

@ -267,6 +267,7 @@ import {
DeleteMessageUpdate,
DeleteStoryUpdate,
Dialog,
FileDownloadLocation,
FileDownloadParameters,
FormattedString,
ForumTopic,
@ -2125,7 +2126,7 @@ export interface TelegramClient extends BaseTelegramClient {
*
* @param params File download parameters
*/
downloadAsBuffer(params: FileDownloadParameters): Promise<Uint8Array>
downloadAsBuffer(location: FileDownloadLocation, params?: FileDownloadParameters): Promise<Uint8Array>
/**
* Download a remote file to a local file (only for NodeJS).
* Promise will resolve once the download is complete.
@ -2135,7 +2136,7 @@ export interface TelegramClient extends BaseTelegramClient {
* @param filename Local file name to which the remote file will be downloaded
* @param params File download parameters
*/
downloadToFile(filename: string, params: FileDownloadParameters): Promise<void>
downloadToFile(filename: string, location: FileDownloadLocation, params?: FileDownloadParameters): Promise<void>
/**
* Download a file and return it as an iterable, which yields file contents
* in chunks of a given size. Order of the chunks is guaranteed to be
@ -2145,7 +2146,7 @@ export interface TelegramClient extends BaseTelegramClient {
*
* @param params Download parameters
*/
downloadAsIterable(params: FileDownloadParameters): AsyncIterableIterator<Uint8Array>
downloadAsIterable(input: FileDownloadLocation, params?: FileDownloadParameters): AsyncIterableIterator<Uint8Array>
/**
* Download a file and return it as a readable stream,
* streaming file contents.
@ -2154,7 +2155,7 @@ export interface TelegramClient extends BaseTelegramClient {
*
* @param params File download parameters
*/
downloadAsStream(params: FileDownloadParameters): ReadableStream<Uint8Array>
downloadAsStream(location: FileDownloadLocation, params?: FileDownloadParameters): ReadableStream<Uint8Array>
/**
* Normalize a {@link InputFileLike} to `InputFile`,
* uploading it if needed.

1
packages/client/src/methods/_imports.ts
View file

@ -36,6 +36,7 @@ import {
DeleteMessageUpdate,
DeleteStoryUpdate,
Dialog,
FileDownloadLocation,
FileDownloadParameters,
FormattedString,
ForumTopic,

11
packages/client/src/methods/files/download-buffer.ts
View file

@ -1,7 +1,7 @@
import { BaseTelegramClient } from '@mtcute/core'
import { concatBuffers } from '@mtcute/core/utils.js'
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
import { downloadAsIterable } from './download-iterable.js'
/**
@ -14,15 +14,16 @@ import { downloadAsIterable } from './download-iterable.js'
*/
export async function downloadAsBuffer(
client: BaseTelegramClient,
params: FileDownloadParameters,
location: FileDownloadLocation,
params?: FileDownloadParameters,
): Promise<Uint8Array> {
if (params.location instanceof FileLocation && ArrayBuffer.isView(params.location.location)) {
return params.location.location
if (location instanceof FileLocation && ArrayBuffer.isView(location.location)) {
return location.location
}
const chunks = []
for await (const chunk of downloadAsIterable(client, params)) {
for await (const chunk of downloadAsIterable(client, location, params)) {
chunks.push(chunk)
}

13
packages/client/src/methods/files/download-file.ts
View file

@ -2,7 +2,7 @@ import { createRequire } from 'module'
import { BaseTelegramClient, MtUnsupportedError } from '@mtcute/core'
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
import { downloadAsIterable } from './download-iterable.js'
let fs: typeof import('fs') | null = null
@ -24,15 +24,16 @@ try {
export async function downloadToFile(
client: BaseTelegramClient,
filename: string,
params: FileDownloadParameters,
location: FileDownloadLocation,
params?: FileDownloadParameters,
): Promise<void> {
if (!fs) {
throw new MtUnsupportedError('Downloading to file is only supported in NodeJS')
}
if (params.location instanceof FileLocation && ArrayBuffer.isView(params.location.location)) {
if (location instanceof FileLocation && ArrayBuffer.isView(location.location)) {
// early return for inline files
const buf = params.location.location
const buf = location.location
return new Promise((resolve, reject) => {
fs!.writeFile(filename, buf, (err) => {
@ -44,7 +45,7 @@ export async function downloadToFile(
const output = fs.createWriteStream(filename)
if (params.abortSignal) {
if (params?.abortSignal) {
params.abortSignal.addEventListener('abort', () => {
client.log.debug('aborting file download %s - cleaning up', filename)
output.destroy()
@ -52,7 +53,7 @@ export async function downloadToFile(
})
}
for await (const chunk of downloadAsIterable(client, params)) {
for await (const chunk of downloadAsIterable(client, location, params)) {
output.write(chunk)
}

20
packages/client/src/methods/files/download-iterable.ts
View file

@ -2,7 +2,7 @@ import { BaseTelegramClient, ConnectionKind, MtArgumentError, MtUnsupportedError
import { ConditionVariable } from '@mtcute/core/utils.js'
import { fileIdToInputFileLocation, fileIdToInputWebFileLocation, parseFileId } from '@mtcute/file-id'
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
import { determinePartSize } from '../../utils/file-utils.js'
// small files (less than 128 kb) are downloaded using the "downloadSmall" pool
@ -20,18 +20,18 @@ const REQUESTS_PER_CONNECTION = 3 // some arbitrary magic value that seems to wo
*/
export async function* downloadAsIterable(
client: BaseTelegramClient,
params: FileDownloadParameters,
input: FileDownloadLocation,
params?: FileDownloadParameters,
): AsyncIterableIterator<Uint8Array> {
const offset = params.offset ?? 0
const offset = params?.offset ?? 0
if (offset % 4096 !== 0) {
throw new MtArgumentError(`Invalid offset: ${offset}. Must be divisible by 4096`)
}
let dcId = params.dcId
let fileSize = params.fileSize
let dcId = params?.dcId
let fileSize = params?.fileSize
const input = params.location
let location: tl.TypeInputFileLocation | tl.TypeInputWebFileLocation
if (input instanceof FileLocation) {
let locationInner = input.location
@ -63,7 +63,7 @@ export async function* downloadAsIterable(
// we will receive a FileMigrateError in case this is invalid
if (!dcId) dcId = client.network.getPrimaryDcId()
const partSizeKb = params.partSize ?? (fileSize ? determinePartSize(fileSize) : 64)
const partSizeKb = params?.partSize ?? (fileSize ? determinePartSize(fileSize) : 64)
if (partSizeKb % 4 !== 0) {
throw new MtArgumentError(`Invalid part size: ${partSizeKb}. Must be divisible by 4.`)
@ -71,7 +71,7 @@ export async function* downloadAsIterable(
const chunkSize = partSizeKb * 1024
let limitBytes = params.limit ?? fileSize ?? Infinity
let limitBytes = params?.limit ?? fileSize ?? Infinity
if (limitBytes === 0) return
let numChunks = limitBytes === Infinity ? Infinity : ~~((limitBytes + chunkSize - offset - 1) / chunkSize)
@ -111,7 +111,7 @@ export async function* downloadAsIterable(
offset: chunkSize * chunk,
limit: chunkSize,
},
{ dcId, kind: connectionKind, abortSignal: params.abortSignal },
{ dcId, kind: connectionKind, abortSignal: params?.abortSignal },
)
} catch (e: unknown) {
if (!tl.RpcError.is(e)) throw e
@ -175,7 +175,7 @@ export async function* downloadAsIterable(
position += buf.length
params.progressCallback?.(position, limitBytes)
params?.progressCallback?.(position, limitBytes)
yield buf

15
packages/client/src/methods/files/download-stream.ts
View file

@ -1,6 +1,6 @@
import { BaseTelegramClient } from '@mtcute/core'
import { FileDownloadParameters, FileLocation } from '../../types/index.js'
import { FileDownloadLocation, FileDownloadParameters, FileLocation } from '../../types/index.js'
import { bufferToStream } from '../../utils/stream-utils.js'
import { downloadAsIterable } from './download-iterable.js'
@ -12,16 +12,17 @@ import { downloadAsIterable } from './download-iterable.js'
*/
export function downloadAsStream(
client: BaseTelegramClient,
params: FileDownloadParameters,
location: FileDownloadLocation,
params?: FileDownloadParameters,
): ReadableStream<Uint8Array> {
if (params.location instanceof FileLocation && ArrayBuffer.isView(params.location.location)) {
return bufferToStream(params.location.location)
if (location instanceof FileLocation && ArrayBuffer.isView(location.location)) {
return bufferToStream(location.location)
}
const cancel = new AbortController()
if (params.abortSignal) {
params.abortSignal.addEventListener('abort', () => {
if (params?.abortSignal) {
params?.abortSignal.addEventListener('abort', () => {
cancel.abort()
})
}
@ -29,7 +30,7 @@ export function downloadAsStream(
return new ReadableStream<Uint8Array>({
start(controller) {
(async () => {
for await (const chunk of downloadAsIterable(client, params)) {
for await (const chunk of downloadAsIterable(client, location, params)) {
controller.enqueue(chunk)
}

4
packages/client/src/types/files/utils.ts
View file

@ -51,13 +51,13 @@ export type InputFileLike =
| tl.TypeInputMedia
| tdFileId.RawFullRemoteFileLocation
export interface FileDownloadParameters {
/**
* File location which should be downloaded.
* You can also provide TDLib and Bot API compatible File ID
*/
location: tl.TypeInputFileLocation | tl.TypeInputWebFileLocation | FileLocation | string
export type FileDownloadLocation = tl.TypeInputFileLocation | tl.TypeInputWebFileLocation | FileLocation | string
export interface FileDownloadParameters {
/**
* Total file size, if known.
* Used to determine upload part size.