Restructure download function
This commit is contained in:
+162
-145
@@ -3,9 +3,9 @@ import { Response } from 'express-serve-static-core'
|
|||||||
import { Asset, ImageSize, KeyType, SharedLink } from '../types'
|
import { Asset, ImageSize, KeyType, SharedLink } from '../types'
|
||||||
import { getConfigOption } from '../config/access'
|
import { getConfigOption } from '../config/access'
|
||||||
import { log } from '../utils/log'
|
import { log } from '../utils/log'
|
||||||
import archiver from 'archiver'
|
import archiver, { Archiver } from 'archiver'
|
||||||
import { sanitize } from '../utils/sanitize'
|
import { sanitize } from '../utils/sanitize'
|
||||||
import { Readable, Transform } from 'stream'
|
import { Readable } from 'stream'
|
||||||
import { pipeline } from 'stream/promises'
|
import { pipeline } from 'stream/promises'
|
||||||
import { promises as fs, createWriteStream } from 'fs'
|
import { promises as fs, createWriteStream } from 'fs'
|
||||||
import { tmpdir } from 'os'
|
import { tmpdir } from 'os'
|
||||||
@@ -13,53 +13,8 @@ import { join } from 'path'
|
|||||||
import { resolveImageEndpoint, ImageEndpoint } from './asset'
|
import { resolveImageEndpoint, ImageEndpoint } from './asset'
|
||||||
import { title } from '../share'
|
import { title } from '../share'
|
||||||
import { getFilename } from '../gallery/filename'
|
import { getFilename } from '../gallery/filename'
|
||||||
|
import { createLimiter } from '../utils/limiter'
|
||||||
/**
|
import { createIdleTimeoutStream } from '../utils/idleTimeoutStream'
|
||||||
* A pass-through Transform that destroys itself if no data flows through for
|
|
||||||
* `idleMs`. The timer is set when the transform is created and reset on every
|
|
||||||
* chunk, so a slow-but-steady download (large video over a slow link) keeps
|
|
||||||
* going, while a genuinely stalled connection still fails fast.
|
|
||||||
*/
|
|
||||||
function createIdleTimeoutStream (idleMs: number): Transform {
|
|
||||||
let timer: NodeJS.Timeout | undefined
|
|
||||||
const transform: Transform = new Transform({
|
|
||||||
transform (chunk, _, cb) {
|
|
||||||
if (timer) clearTimeout(timer)
|
|
||||||
timer = setTimeout(() => transform.destroy(new Error(`No data received for ${idleMs}ms`)), idleMs)
|
|
||||||
cb(null, chunk)
|
|
||||||
},
|
|
||||||
flush (cb) {
|
|
||||||
if (timer) clearTimeout(timer)
|
|
||||||
cb()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
// Arm the timer immediately so a response that returns headers but never
|
|
||||||
// sends a body also times out.
|
|
||||||
timer = setTimeout(() => transform.destroy(new Error(`No data received for ${idleMs}ms`)), idleMs)
|
|
||||||
return transform
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns a function that runs at most `limit` async tasks concurrently.
|
|
||||||
* Tasks queue when the limit is reached and resume in FIFO order.
|
|
||||||
*/
|
|
||||||
function createLimiter (limit: number) {
|
|
||||||
let active = 0
|
|
||||||
const queue: Array<() => void> = []
|
|
||||||
return async function run<T> (fn: () => Promise<T>): Promise<T> {
|
|
||||||
if (active >= limit) {
|
|
||||||
await new Promise<void>(resolve => queue.push(resolve))
|
|
||||||
}
|
|
||||||
active++
|
|
||||||
try {
|
|
||||||
return await fn()
|
|
||||||
} finally {
|
|
||||||
active--
|
|
||||||
const next = queue.shift()
|
|
||||||
if (next) next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Download all assets in a share as a zip file.
|
* Download all assets in a share as a zip file.
|
||||||
@@ -68,26 +23,41 @@ export async function downloadAll (res: Response, share: SharedLink) {
|
|||||||
await downloadAssets(res, share, share.assets)
|
await downloadAssets(res, share, share.assets)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type StagedAsset = { tempfile: string, asset: Asset, endpoint: ImageEndpoint }
|
||||||
|
type Failure = { asset: Asset, url: string, status?: number, error?: unknown }
|
||||||
|
type StageOutcome = StagedAsset | { failure: Failure } | null
|
||||||
|
|
||||||
|
type AbortFlag = { aborted: boolean }
|
||||||
|
|
||||||
|
type StagingOptions = {
|
||||||
|
stagingDir: string
|
||||||
|
concurrency: number
|
||||||
|
maxAttempts: number
|
||||||
|
headerTimeoutMs: number
|
||||||
|
idleTimeoutMs: number
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stream the given assets back as a zip file.
|
* Stream the given assets back as a zip file.
|
||||||
*
|
*
|
||||||
* Pattern adapted from Immich's own download service:
|
* Immich's own download service (server/src/services/download.service.ts)
|
||||||
* https://github.com/immich-app/immich/blob/main/server/src/services/download.service.ts
|
* zips files directly from local disk - no HTTP, retries, or timeouts needed.
|
||||||
|
* We're a proxy fetching over HTTP, so the shape diverges:
|
||||||
*
|
*
|
||||||
* Per asset, the upstream fetch is two phases:
|
* - Bounded concurrency against the upstream Immich server.
|
||||||
* 1. Get the response headers (retried up to 3 times with linear backoff,
|
* - Two-phase fetch per asset: get headers (retried with linear backoff,
|
||||||
* bounded by a 20s header-receipt timeout).
|
* bounded by a 20s header-receipt timeout), then stream the body to a
|
||||||
* 2. Stream the body to a temp file, guarded by an idle timeout that
|
* temp file under an idle timeout that resets on every chunk -
|
||||||
* resets on every chunk. A genuinely stalled connection fails fast; a
|
* slow-but-steady downloads survive, truly stalled ones fail fast.
|
||||||
* slow-but-steady transfer (large video over a slow link) completes.
|
* - Stage everything to disk first so we can detect failure BEFORE any
|
||||||
|
* bytes hit the client. Once we've started streaming the zip we can't
|
||||||
|
* recover gracefully; aborting means killing the response socket and
|
||||||
|
* leaving a visibly broken download. This is the alternative to silently
|
||||||
|
* omitting failed assets - the user gets a clear "broken" signal instead
|
||||||
|
* of a zip quietly missing files.
|
||||||
*
|
*
|
||||||
* If any asset ultimately fails, the in-flight download is aborted mid-stream:
|
* The only thing we share with Immich here is using zip STORE (no
|
||||||
* the archive is aborted, the HTTP response socket is destroyed, and the
|
* compression), since photos and videos are already compressed.
|
||||||
* failure is logged. We do this (rather than silently omitting the asset)
|
|
||||||
* because by the time we know the asset failed we've already sent a 200
|
|
||||||
* response to the client, so the only way to signal "your zip is incomplete"
|
|
||||||
* is to terminate the response - leaving the user with a visibly broken/
|
|
||||||
* truncated download instead of a zip that's quietly missing files.
|
|
||||||
*/
|
*/
|
||||||
export async function downloadAssets (res: Response, share: SharedLink, assets: Asset[]) {
|
export async function downloadAssets (res: Response, share: SharedLink, assets: Asset[]) {
|
||||||
res.setHeader('Content-Type', 'application/zip')
|
res.setHeader('Content-Type', 'application/zip')
|
||||||
@@ -96,25 +66,90 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${filename}`)
|
res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${filename}`)
|
||||||
// Hint to intermediate proxies (Nginx, etc.) not to buffer this response.
|
// Hint to intermediate proxies (Nginx, etc.) not to buffer this response.
|
||||||
res.setHeader('X-Accel-Buffering', 'no')
|
res.setHeader('X-Accel-Buffering', 'no')
|
||||||
// STORE rather than deflate - photos and videos are already compressed,
|
|
||||||
// this is the same as Immich does
|
|
||||||
const archive = archiver('zip', { store: true })
|
const archive = archiver('zip', { store: true })
|
||||||
archive.pipe(res)
|
archive.pipe(res)
|
||||||
|
|
||||||
const maxAttempts = 3
|
const options: StagingOptions = {
|
||||||
const headerTimeoutMs = 20_000
|
stagingDir: await fs.mkdtemp(join(tmpdir(), 'ipp-zip-')),
|
||||||
const idleTimeoutMs = 20_000
|
concurrency: Math.max(1, getConfigOption('ipp.downloadFromImmichConcurrencyLimit', 8) as number),
|
||||||
const concurrency = Math.max(1, getConfigOption('ipp.downloadFromImmichConcurrencyLimit', 8) as number)
|
maxAttempts: 3,
|
||||||
|
headerTimeoutMs: 20_000,
|
||||||
|
idleTimeoutMs: 20_000
|
||||||
|
}
|
||||||
|
const abortFlag: AbortFlag = { aborted: false }
|
||||||
|
|
||||||
const stagingDir = await fs.mkdtemp(join(tmpdir(), 'ipp-zip-'))
|
try {
|
||||||
let aborted = false
|
const stages = stageAssetsToDisk(assets, options, abortFlag)
|
||||||
|
const failure = await archiveStaged(archive, stages, abortFlag)
|
||||||
|
if (failure) {
|
||||||
|
abortDownload(archive, res, share, failure)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// finalize() resolves when archiver has finished writing the zip output,
|
||||||
|
// which means every queued tempfile has been read. Safe to delete after.
|
||||||
|
await archive.finalize()
|
||||||
|
} finally {
|
||||||
|
await fs.rm(options.stagingDir, { recursive: true, force: true }).catch(() => { /* best effort */ })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type StagedAsset = { tempfile: string, asset: Asset, endpoint: ImageEndpoint }
|
/**
|
||||||
type Failure = { asset: Asset, url: string, status?: number, error?: unknown }
|
* Kick off staging for every asset at once; the limiter caps how many run
|
||||||
|
* concurrently. Returns the promise array in input order so the consumer can
|
||||||
|
* archive in order.
|
||||||
|
*/
|
||||||
|
function stageAssetsToDisk (assets: Asset[], options: StagingOptions, abortFlag: AbortFlag): Promise<StageOutcome>[] {
|
||||||
|
const limit = createLimiter(options.concurrency)
|
||||||
|
return assets.map((asset, index) => limit(() => stageOne(asset, index, options, abortFlag)))
|
||||||
|
}
|
||||||
|
|
||||||
const stageOne = async (asset: Asset, index: number): Promise<StagedAsset | { failure: Failure } | null> => {
|
/**
|
||||||
// Skip work if another stage has already triggered an abort.
|
* Walk the staged assets in input order, adding each to the archive as soon
|
||||||
if (aborted) return null
|
* as it lands. The first failure sets the abort flag (so unstarted stages
|
||||||
|
* bail out) and is returned to the caller. We wait for in-flight stages to
|
||||||
|
* settle before returning so the staging dir is safe to delete.
|
||||||
|
*/
|
||||||
|
async function archiveStaged (archive: Archiver, stages: Promise<StageOutcome>[], abortFlag: AbortFlag): Promise<Failure | null> {
|
||||||
|
let failure: Failure | null = null
|
||||||
|
for (const stage of stages) {
|
||||||
|
const result = await stage
|
||||||
|
if (result === null) break // aborted by an earlier stage
|
||||||
|
if ('failure' in result) {
|
||||||
|
abortFlag.aborted = true
|
||||||
|
failure = result.failure
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// archive.file queues the entry; archiver lazily opens and reads it.
|
||||||
|
archive.file(result.tempfile, { name: getFilename(result.asset, result.endpoint.servedSize) })
|
||||||
|
}
|
||||||
|
|
||||||
|
// After abort, wait for any in-flight stages to settle before we delete
|
||||||
|
// the staging dir. stageOne never rejects (errors become failure objects),
|
||||||
|
// so a plain Promise.all is safe.
|
||||||
|
if (abortFlag.aborted) await Promise.all(stages)
|
||||||
|
return failure
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortDownload (archive: Archiver, res: Response, share: SharedLink, failure: Failure) {
|
||||||
|
const detail = failure.status !== undefined
|
||||||
|
? `HTTP ${failure.status}`
|
||||||
|
: (failure.error instanceof Error ? failure.error.message : String(failure.error))
|
||||||
|
log(`Aborting zip download for share ${share.key}: failed to fetch asset ${failure.asset.id} from ${failure.url} (${detail})`)
|
||||||
|
archive.abort()
|
||||||
|
res.destroy()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch one asset's upstream bytes into a temp file. Two phases:
|
||||||
|
* 1. fetchHeadersWithRetry - get the response headers, retried on
|
||||||
|
* transient failure.
|
||||||
|
* 2. streamBodyToTempFile - drain the body to disk under an idle timeout.
|
||||||
|
*
|
||||||
|
* Returns the staged asset on success, a wrapped Failure on error, or null
|
||||||
|
* if we observed the abort flag and bailed without doing work.
|
||||||
|
*/
|
||||||
|
async function stageOne (asset: Asset, index: number, options: StagingOptions, abortFlag: AbortFlag): Promise<StageOutcome> {
|
||||||
|
if (abortFlag.aborted) return null
|
||||||
|
|
||||||
const endpoint = resolveImageEndpoint(ImageSize.original, asset)
|
const endpoint = resolveImageEndpoint(ImageSize.original, asset)
|
||||||
const url = buildUrl(apiUrl() + '/assets/' + encodeURIComponent(asset.id) + endpoint.subpath, {
|
const url = buildUrl(apiUrl() + '/assets/' + encodeURIComponent(asset.id) + endpoint.subpath, {
|
||||||
@@ -123,27 +158,52 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
})
|
})
|
||||||
const reqAuthHeaders = await authHeaders(asset.keyType || KeyType.key, asset.key, asset.password)
|
const reqAuthHeaders = await authHeaders(asset.keyType || KeyType.key, asset.key, asset.password)
|
||||||
|
|
||||||
// Phase 1: get response headers, retrying on transient failure.
|
const fetched = await fetchHeadersWithRetry(url, reqAuthHeaders, options.maxAttempts, options.headerTimeoutMs, abortFlag, asset)
|
||||||
// We use AbortController + a clearable timer rather than
|
if (fetched === null) return null
|
||||||
// `AbortSignal.timeout` because the signal we pass to fetch stays bound
|
if ('failure' in fetched) return { failure: { ...fetched.failure, asset, url } }
|
||||||
// to the response body - if the timeout fires after headers arrive but
|
|
||||||
// while the body is still streaming, the body read errors out. The
|
// Use the array index in the path so we never collide on duplicate IDs.
|
||||||
// header timer is cleared as soon as we have a response; from there the
|
const tempfile = join(options.stagingDir, `${index}-${asset.id}`)
|
||||||
// idle-timeout transform downstream guards against stalled bodies.
|
const streamed = await streamBodyToTempFile(fetched.response, tempfile, options.idleTimeoutMs)
|
||||||
let response: globalThis.Response | undefined
|
if ('failure' in streamed) return { failure: { asset, url, error: streamed.failure } }
|
||||||
|
|
||||||
|
return { tempfile, asset, endpoint }
|
||||||
|
}
|
||||||
|
|
||||||
|
type HeaderFetchOutcome =
|
||||||
|
| { response: globalThis.Response }
|
||||||
|
| { failure: { status?: number, error?: unknown } }
|
||||||
|
| null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET `url` until we have response headers or run out of attempts. Retries
|
||||||
|
* use linear backoff to avoid hammering a struggling upstream.
|
||||||
|
*
|
||||||
|
* AbortController + clearable timer (rather than `AbortSignal.timeout`)
|
||||||
|
* because the signal we pass to fetch stays bound to the response body - if
|
||||||
|
* the timeout fires after headers arrive but while the body is still
|
||||||
|
* streaming, the body read errors out. We clear the header timer as soon
|
||||||
|
* as we have a response; downstream the idle-timeout transform guards
|
||||||
|
* against stalled bodies.
|
||||||
|
*/
|
||||||
|
async function fetchHeadersWithRetry (
|
||||||
|
url: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
maxAttempts: number,
|
||||||
|
headerTimeoutMs: number,
|
||||||
|
abortFlag: AbortFlag,
|
||||||
|
asset: Asset
|
||||||
|
): Promise<HeaderFetchOutcome> {
|
||||||
let lastStatus: number | undefined
|
let lastStatus: number | undefined
|
||||||
let lastError: unknown
|
let lastError: unknown
|
||||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
if (aborted) return null
|
if (abortFlag.aborted) return null
|
||||||
const controller = new AbortController()
|
const controller = new AbortController()
|
||||||
const headerTimer = setTimeout(() => controller.abort(new Error(`No response headers within ${headerTimeoutMs}ms`)), headerTimeoutMs)
|
const headerTimer = setTimeout(() => controller.abort(new Error(`No response headers within ${headerTimeoutMs}ms`)), headerTimeoutMs)
|
||||||
try {
|
try {
|
||||||
const data = await fetch(url, { signal: controller.signal, headers: reqAuthHeaders })
|
const data = await fetch(url, { signal: controller.signal, headers })
|
||||||
clearTimeout(headerTimer)
|
clearTimeout(headerTimer)
|
||||||
if (data.ok) {
|
if (data.ok) return { response: data }
|
||||||
response = data
|
|
||||||
break
|
|
||||||
}
|
|
||||||
await data.body?.cancel()
|
await data.body?.cancel()
|
||||||
lastStatus = data.status
|
lastStatus = data.status
|
||||||
lastError = undefined
|
lastError = undefined
|
||||||
@@ -157,18 +217,19 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
? `HTTP ${lastStatus}`
|
? `HTTP ${lastStatus}`
|
||||||
: (lastError instanceof Error ? lastError.message : String(lastError))
|
: (lastError instanceof Error ? lastError.message : String(lastError))
|
||||||
log(`Retrying asset ${asset.id} (attempt ${attempt + 1}/${maxAttempts}) after ${reason}`)
|
log(`Retrying asset ${asset.id} (attempt ${attempt + 1}/${maxAttempts}) after ${reason}`)
|
||||||
// Linear backoff between attempts to avoid hammering a struggling server.
|
|
||||||
await new Promise(resolve => setTimeout(resolve, 500 * attempt))
|
await new Promise(resolve => setTimeout(resolve, 500 * attempt))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return { failure: { status: lastStatus, error: lastError } }
|
||||||
if (!response || !response.body) {
|
|
||||||
return { failure: { asset, url, status: lastStatus, error: lastError } }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Phase 2: stream body to a temp file, guarded by an idle timeout.
|
/**
|
||||||
// Use the array index in the path so we never collide on duplicate IDs.
|
* Stream `response.body` to `tempfile`, guarded by an idle timeout. The
|
||||||
const tempfile = join(stagingDir, `${index}-${asset.id}`)
|
* idle stream destroys itself if no chunk arrives within `idleMs`, killing
|
||||||
|
* the pipeline with a clear error.
|
||||||
|
*/
|
||||||
|
async function streamBodyToTempFile (response: globalThis.Response, tempfile: string, idleMs: number): Promise<{ ok: true } | { failure: unknown }> {
|
||||||
|
if (!response.body) return { failure: new Error('Upstream response has no body') }
|
||||||
// `response.body` is the global/undici ReadableStream<Uint8Array>;
|
// `response.body` is the global/undici ReadableStream<Uint8Array>;
|
||||||
// Readable.fromWeb expects node:stream/web's ReadableStream<any>. The
|
// Readable.fromWeb expects node:stream/web's ReadableStream<any>. The
|
||||||
// two are structurally compatible at runtime but TS sees them as
|
// two are structurally compatible at runtime but TS sees them as
|
||||||
@@ -176,53 +237,9 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
const body = Readable.fromWeb(response.body as any)
|
const body = Readable.fromWeb(response.body as any)
|
||||||
try {
|
try {
|
||||||
await pipeline(body, createIdleTimeoutStream(idleTimeoutMs), createWriteStream(tempfile))
|
await pipeline(body, createIdleTimeoutStream(idleMs), createWriteStream(tempfile))
|
||||||
|
return { ok: true }
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return { failure: { asset, url, error: e } }
|
return { failure: e }
|
||||||
}
|
|
||||||
|
|
||||||
return { tempfile, asset, endpoint }
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const limit = createLimiter(concurrency)
|
|
||||||
// Kick off every stage at once; the limiter caps how many run concurrently.
|
|
||||||
// The map preserves input order so the consumer below archives in order.
|
|
||||||
const stages = assets.map((asset, index) => limit(() => stageOne(asset, index)))
|
|
||||||
|
|
||||||
let failure: Failure | null = null
|
|
||||||
for (const stage of stages) {
|
|
||||||
const result = await stage
|
|
||||||
if (result === null) break // aborted by an earlier stage
|
|
||||||
if ('failure' in result) {
|
|
||||||
aborted = true
|
|
||||||
failure = result.failure
|
|
||||||
break
|
|
||||||
}
|
|
||||||
// archive.file queues the entry; archiver lazily opens and reads it.
|
|
||||||
archive.file(result.tempfile, { name: getFilename(result.asset, result.endpoint.servedSize) })
|
|
||||||
}
|
|
||||||
|
|
||||||
// After abort, wait for any in-flight stages to settle before we delete
|
|
||||||
// the staging dir. stageOne never rejects (errors become failure objects),
|
|
||||||
// so a plain Promise.all is safe.
|
|
||||||
if (aborted) await Promise.all(stages)
|
|
||||||
|
|
||||||
if (failure) {
|
|
||||||
const f = failure as Failure
|
|
||||||
const detail = f.status !== undefined
|
|
||||||
? `HTTP ${f.status}`
|
|
||||||
: (f.error instanceof Error ? f.error.message : String(f.error))
|
|
||||||
log(`Aborting zip download for share ${share.key}: failed to fetch asset ${f.asset.id} from ${f.url} (${detail})`)
|
|
||||||
archive.abort()
|
|
||||||
res.destroy()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// finalize() resolves when archiver has finished writing the zip output,
|
|
||||||
// which means every queued tempfile has been read. Safe to delete after.
|
|
||||||
await archive.finalize()
|
|
||||||
} finally {
|
|
||||||
await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => { /* best effort */ })
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import { Transform } from 'stream'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A pass-through Transform that destroys itself if no data flows through for
|
||||||
|
* `idleMs`. The timer is set when the transform is created and reset on every
|
||||||
|
* chunk, so a slow-but-steady download (large video over a slow link) keeps
|
||||||
|
* going, while a genuinely stalled connection still fails fast.
|
||||||
|
*/
|
||||||
|
export function createIdleTimeoutStream (idleMs: number): Transform {
|
||||||
|
let timer: NodeJS.Timeout | undefined
|
||||||
|
const transform: Transform = new Transform({
|
||||||
|
transform (chunk, _, cb) {
|
||||||
|
if (timer) clearTimeout(timer)
|
||||||
|
timer = setTimeout(() => transform.destroy(new Error(`No data received for ${idleMs}ms`)), idleMs)
|
||||||
|
cb(null, chunk)
|
||||||
|
},
|
||||||
|
flush (cb) {
|
||||||
|
if (timer) clearTimeout(timer)
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
// Arm the timer immediately so a response that returns headers but never
|
||||||
|
// sends a body also times out.
|
||||||
|
timer = setTimeout(() => transform.destroy(new Error(`No data received for ${idleMs}ms`)), idleMs)
|
||||||
|
return transform
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
/**
|
||||||
|
* Returns a function that runs at most `limit` async tasks concurrently.
|
||||||
|
* Tasks queue when the limit is reached and resume in FIFO order.
|
||||||
|
*/
|
||||||
|
export function createLimiter (limit: number) {
|
||||||
|
let active = 0
|
||||||
|
const queue: Array<() => void> = []
|
||||||
|
return async function run<T> (fn: () => Promise<T>): Promise<T> {
|
||||||
|
if (active >= limit) {
|
||||||
|
await new Promise<void>(resolve => queue.push(resolve))
|
||||||
|
}
|
||||||
|
active++
|
||||||
|
try {
|
||||||
|
return await fn()
|
||||||
|
} finally {
|
||||||
|
active--
|
||||||
|
const next = queue.shift()
|
||||||
|
if (next) next()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user