#284 fix issue where aborted downloads are not cleaned up
This commit is contained in:
+51
-21
@@ -52,8 +52,6 @@ type StagedAsset = { tempfile: string, asset: Asset, endpoint: ImageEndpoint }
|
|||||||
type Failure = { asset: Asset, url: string, status?: number, error?: unknown }
|
type Failure = { asset: Asset, url: string, status?: number, error?: unknown }
|
||||||
type StageOutcome = StagedAsset | { failure: Failure } | null
|
type StageOutcome = StagedAsset | { failure: Failure } | null
|
||||||
|
|
||||||
type AbortFlag = { aborted: boolean }
|
|
||||||
|
|
||||||
type StagingOptions = {
|
type StagingOptions = {
|
||||||
stagingDir: string
|
stagingDir: string
|
||||||
concurrency: number
|
concurrency: number
|
||||||
@@ -92,6 +90,8 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
// Hint to intermediate proxies (Nginx, etc.) not to buffer this response.
|
// Hint to intermediate proxies (Nginx, etc.) not to buffer this response.
|
||||||
res.setHeader('X-Accel-Buffering', 'no')
|
res.setHeader('X-Accel-Buffering', 'no')
|
||||||
const archive = archiver('zip', { store: true })
|
const archive = archiver('zip', { store: true })
|
||||||
|
// Without a listener, an archiver 'error' emission would crash the process.
|
||||||
|
archive.on('error', e => log(`Archiver error for share ${share.key}: ${e.message}`))
|
||||||
archive.pipe(res)
|
archive.pipe(res)
|
||||||
|
|
||||||
const options: StagingOptions = {
|
const options: StagingOptions = {
|
||||||
@@ -101,18 +101,43 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
headerTimeoutMs: 20_000,
|
headerTimeoutMs: 20_000,
|
||||||
idleTimeoutMs: 20_000
|
idleTimeoutMs: 20_000
|
||||||
}
|
}
|
||||||
const abortFlag: AbortFlag = { aborted: false }
|
|
||||||
|
const controller = new AbortController()
|
||||||
|
let clientGone = false
|
||||||
|
let resolveClosed!: () => void
|
||||||
|
const resClosed = new Promise<void>(resolve => { resolveClosed = resolve })
|
||||||
|
res.once('close', () => {
|
||||||
|
if (res.writableFinished) return
|
||||||
|
clientGone = true
|
||||||
|
controller.abort()
|
||||||
|
resolveClosed()
|
||||||
|
})
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const stages = stageAssetsToDisk(assets, options, abortFlag)
|
const stages = stageAssetsToDisk(assets, options, controller.signal)
|
||||||
const failure = await archiveStaged(archive, stages, abortFlag)
|
const failure = await archiveStaged(archive, stages, controller)
|
||||||
|
if (clientGone) {
|
||||||
|
log(`Zip download for share ${share.key} cancelled by client`)
|
||||||
|
archive.abort()
|
||||||
|
return
|
||||||
|
}
|
||||||
if (failure) {
|
if (failure) {
|
||||||
abortDownload(archive, res, share, failure)
|
abortDownload(archive, res, share, failure)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// finalize() resolves when archiver has finished writing the zip output,
|
// finalize() resolves when archiver has finished writing the zip output,
|
||||||
// which means every queued tempfile has been read. Safe to delete after.
|
// which means every queued tempfile has been read. Safe to delete after.
|
||||||
await archive.finalize()
|
// Raced against client disconnect because finalize() never settles once
|
||||||
|
// the response is destroyed. The inline rejection handler also stops a
|
||||||
|
// late finalize failure becoming an unhandled rejection after a lost race.
|
||||||
|
const finished = archive.finalize().then(() => 'done' as const, () => 'error' as const)
|
||||||
|
const outcome = await Promise.race([finished, resClosed.then(() => 'closed' as const)])
|
||||||
|
if (outcome !== 'done') {
|
||||||
|
if (outcome === 'closed') log(`Zip download for share ${share.key} cancelled by client`)
|
||||||
|
// 'error' was already logged by the archiver error listener
|
||||||
|
archive.abort()
|
||||||
|
res.destroy()
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
await fs.rm(options.stagingDir, { recursive: true, force: true }).catch(() => { /* best effort */ })
|
await fs.rm(options.stagingDir, { recursive: true, force: true }).catch(() => { /* best effort */ })
|
||||||
}
|
}
|
||||||
@@ -123,24 +148,25 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
* concurrently. Returns the promise array in input order so the consumer can
|
* concurrently. Returns the promise array in input order so the consumer can
|
||||||
* archive in order.
|
* archive in order.
|
||||||
*/
|
*/
|
||||||
function stageAssetsToDisk (assets: Asset[], options: StagingOptions, abortFlag: AbortFlag): Promise<StageOutcome>[] {
|
function stageAssetsToDisk (assets: Asset[], options: StagingOptions, signal: AbortSignal): Promise<StageOutcome>[] {
|
||||||
const limit = createLimiter(options.concurrency)
|
const limit = createLimiter(options.concurrency)
|
||||||
return assets.map((asset, index) => limit(() => stageOne(asset, index, options, abortFlag)))
|
return assets.map((asset, index) => limit(() => stageOne(asset, index, options, signal)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Walk the staged assets in input order, adding each to the archive as soon
|
* Walk the staged assets in input order, adding each to the archive as soon
|
||||||
* as it lands. The first failure sets the abort flag (so unstarted stages
|
* as it lands. The first failure aborts the download signal (so unstarted
|
||||||
* bail out) and is returned to the caller. We wait for in-flight stages to
|
* stages bail out and in-flight fetches are cancelled) and is returned to the
|
||||||
* settle before returning so the staging dir is safe to delete.
|
* caller. We wait for in-flight stages to settle before returning so the
|
||||||
|
* staging dir is safe to delete.
|
||||||
*/
|
*/
|
||||||
async function archiveStaged (archive: Archiver, stages: Promise<StageOutcome>[], abortFlag: AbortFlag): Promise<Failure | null> {
|
async function archiveStaged (archive: Archiver, stages: Promise<StageOutcome>[], controller: AbortController): Promise<Failure | null> {
|
||||||
let failure: Failure | null = null
|
let failure: Failure | null = null
|
||||||
for (const stage of stages) {
|
for (const stage of stages) {
|
||||||
const result = await stage
|
const result = await stage
|
||||||
if (result === null) break // aborted by an earlier stage
|
if (result === null) break // aborted by an earlier stage
|
||||||
if ('failure' in result) {
|
if ('failure' in result) {
|
||||||
abortFlag.aborted = true
|
controller.abort()
|
||||||
failure = result.failure
|
failure = result.failure
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -151,7 +177,7 @@ async function archiveStaged (archive: Archiver, stages: Promise<StageOutcome>[]
|
|||||||
// After abort, wait for any in-flight stages to settle before we delete
|
// After abort, wait for any in-flight stages to settle before we delete
|
||||||
// the staging dir. stageOne never rejects (errors become failure objects),
|
// the staging dir. stageOne never rejects (errors become failure objects),
|
||||||
// so a plain Promise.all is safe.
|
// so a plain Promise.all is safe.
|
||||||
if (abortFlag.aborted) await Promise.all(stages)
|
if (controller.signal.aborted) await Promise.all(stages)
|
||||||
return failure
|
return failure
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -173,14 +199,14 @@ function abortDownload (archive: Archiver, res: Response, share: SharedLink, fai
|
|||||||
* Returns the staged asset on success, a wrapped Failure on error, or null
|
* Returns the staged asset on success, a wrapped Failure on error, or null
|
||||||
* if we observed the abort flag and bailed without doing work.
|
* if we observed the abort flag and bailed without doing work.
|
||||||
*/
|
*/
|
||||||
async function stageOne (asset: Asset, index: number, options: StagingOptions, abortFlag: AbortFlag): Promise<StageOutcome> {
|
async function stageOne (asset: Asset, index: number, options: StagingOptions, signal: AbortSignal): Promise<StageOutcome> {
|
||||||
if (abortFlag.aborted) return null
|
if (signal.aborted) return null
|
||||||
|
|
||||||
const endpoint = resolveImageEndpoint(ImageSize.original, asset)
|
const endpoint = resolveImageEndpoint(ImageSize.original, asset)
|
||||||
const url = assetFetchUrl(asset, endpoint.subpath, endpoint.sizeQueryParam)
|
const url = assetFetchUrl(asset, endpoint.subpath, endpoint.sizeQueryParam)
|
||||||
const reqAuthHeaders = await authHeadersForAsset(asset)
|
const reqAuthHeaders = await authHeadersForAsset(asset)
|
||||||
|
|
||||||
const fetched = await fetchHeadersWithRetry(url, reqAuthHeaders, options.maxAttempts, options.headerTimeoutMs, abortFlag, asset)
|
const fetched = await fetchHeadersWithRetry(url, reqAuthHeaders, options.maxAttempts, options.headerTimeoutMs, signal, asset)
|
||||||
if (fetched === null) return null
|
if (fetched === null) return null
|
||||||
if ('failure' in fetched) return { failure: { ...fetched.failure, asset, url } }
|
if ('failure' in fetched) return { failure: { ...fetched.failure, asset, url } }
|
||||||
|
|
||||||
@@ -250,23 +276,27 @@ type HeaderFetchOutcome =
|
|||||||
* streaming, the body read errors out. We clear the header timer as soon
|
* streaming, the body read errors out. We clear the header timer as soon
|
||||||
* as we have a response; downstream the idle-timeout transform guards
|
* as we have a response; downstream the idle-timeout transform guards
|
||||||
* against stalled bodies.
|
* against stalled bodies.
|
||||||
|
*
|
||||||
|
* The download-wide `signal` is combined into the fetch signal so that an
|
||||||
|
* aborted download (asset failure or client disconnect) cancels the header
|
||||||
|
* wait and any in-flight body stream immediately.
|
||||||
*/
|
*/
|
||||||
async function fetchHeadersWithRetry (
|
async function fetchHeadersWithRetry (
|
||||||
url: string,
|
url: string,
|
||||||
headers: Record<string, string>,
|
headers: Record<string, string>,
|
||||||
maxAttempts: number,
|
maxAttempts: number,
|
||||||
headerTimeoutMs: number,
|
headerTimeoutMs: number,
|
||||||
abortFlag: AbortFlag,
|
signal: AbortSignal,
|
||||||
asset: Asset
|
asset: Asset
|
||||||
): Promise<HeaderFetchOutcome> {
|
): Promise<HeaderFetchOutcome> {
|
||||||
let lastStatus: number | undefined
|
let lastStatus: number | undefined
|
||||||
let lastError: unknown
|
let lastError: unknown
|
||||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
if (abortFlag.aborted) return null
|
if (signal.aborted) return null
|
||||||
const controller = new AbortController()
|
const controller = new AbortController()
|
||||||
const headerTimer = setTimeout(() => controller.abort(new Error(`No response headers within ${headerTimeoutMs}ms`)), headerTimeoutMs)
|
const headerTimer = setTimeout(() => controller.abort(new Error(`No response headers within ${headerTimeoutMs}ms`)), headerTimeoutMs)
|
||||||
try {
|
try {
|
||||||
const data = await fetch(url, { signal: controller.signal, headers })
|
const data = await fetch(url, { signal: AbortSignal.any([controller.signal, signal]), headers })
|
||||||
clearTimeout(headerTimer)
|
clearTimeout(headerTimer)
|
||||||
if (data.ok) return { response: data }
|
if (data.ok) return { response: data }
|
||||||
await data.body?.cancel()
|
await data.body?.cancel()
|
||||||
@@ -277,7 +307,7 @@ async function fetchHeadersWithRetry (
|
|||||||
lastError = e
|
lastError = e
|
||||||
lastStatus = undefined
|
lastStatus = undefined
|
||||||
}
|
}
|
||||||
if (attempt < maxAttempts) {
|
if (attempt < maxAttempts && !signal.aborted) {
|
||||||
const reason = lastStatus !== undefined
|
const reason = lastStatus !== undefined
|
||||||
? `HTTP ${lastStatus}`
|
? `HTTP ${lastStatus}`
|
||||||
: (lastError instanceof Error ? lastError.message : String(lastError))
|
: (lastError instanceof Error ? lastError.message : String(lastError))
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
import { describe, it, expect, vi, beforeAll, afterAll, afterEach } from 'vitest'
|
||||||
|
import { promises as fs } from 'fs'
|
||||||
|
import { Writable } from 'stream'
|
||||||
|
import type { Response } from 'express-serve-static-core'
|
||||||
|
import { downloadAssets } from '../src/stream/download'
|
||||||
|
import { Asset, AssetType, KeyType, SharedLink } from '../src/types'
|
||||||
|
|
||||||
|
/*
|
||||||
|
Isolate this suite's staging dirs in a dedicated tmp dir so the
|
||||||
|
before/after assertions can't race other test files that also create
|
||||||
|
ipp-zip-* dirs under the real os.tmpdir().
|
||||||
|
*/
|
||||||
|
const { TEST_TMP } = vi.hoisted(() => {
|
||||||
|
return { TEST_TMP: (process.env.TMPDIR || '/tmp') + '/ipp-download-abort-test-' + process.pid }
|
||||||
|
})
|
||||||
|
vi.mock('os', async importOriginal => {
|
||||||
|
const orig = await importOriginal<typeof import('os')>()
|
||||||
|
return { ...orig, tmpdir: () => TEST_TMP }
|
||||||
|
})
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
await fs.mkdir(TEST_TMP, { recursive: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await fs.rm(TEST_TMP, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals()
|
||||||
|
})
|
||||||
|
|
||||||
|
async function stagingDirs (): Promise<string[]> {
|
||||||
|
const entries = await fs.readdir(TEST_TMP).catch(() => [] as string[])
|
||||||
|
return entries.filter(name => name.startsWith('ipp-zip-'))
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeAsset (id: string): Asset {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
key: 'testkey',
|
||||||
|
keyType: KeyType.key,
|
||||||
|
type: AssetType.image,
|
||||||
|
isTrashed: false,
|
||||||
|
originalFileName: id + '.jpg',
|
||||||
|
originalMimeType: 'image/jpeg'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const share: SharedLink = {
|
||||||
|
key: 'testkey',
|
||||||
|
keyType: KeyType.key,
|
||||||
|
type: 'ALBUM',
|
||||||
|
description: 'Test album',
|
||||||
|
assets: []
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
Minimal stand-in for the Express response: a real Writable (so 'close',
|
||||||
|
writableFinished and destroy behave like the real thing) plus the header
|
||||||
|
setters downloadAssets calls.
|
||||||
|
*/
|
||||||
|
class FakeRes extends Writable {
|
||||||
|
received = 0
|
||||||
|
|
||||||
|
constructor () {
|
||||||
|
super()
|
||||||
|
// Node's http stack handles response stream errors internally; a bare
|
||||||
|
// Writable would crash the test process on destroy(err) instead.
|
||||||
|
this.on('error', () => { /* swallowed, like http.ServerResponse */ })
|
||||||
|
}
|
||||||
|
|
||||||
|
setHeader (_name: string, _value: string) { return this }
|
||||||
|
|
||||||
|
_write (chunk: Buffer, _enc: string, cb: (error?: Error | null) => void) {
|
||||||
|
this.received += chunk.length
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function asResponse (res: FakeRes): Response {
|
||||||
|
return res as unknown as Response
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Fetch stub that returns the full body immediately. */
|
||||||
|
function instantFetch (bytes: number) {
|
||||||
|
return vi.fn(async () => new globalThis.Response(new Uint8Array(bytes), { status: 200 }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch stub whose body sends one chunk then stays open forever, erroring
|
||||||
|
* only when the passed-in signal aborts - like a real streaming download
|
||||||
|
* that gets cancelled. Records each request's signal for assertions.
|
||||||
|
*/
|
||||||
|
function trickleFetch (signals: AbortSignal[]) {
|
||||||
|
return vi.fn(async (_url: unknown, init?: RequestInit) => {
|
||||||
|
const signal = init?.signal as AbortSignal
|
||||||
|
signals.push(signal)
|
||||||
|
const stream = new ReadableStream({
|
||||||
|
start (controller) {
|
||||||
|
controller.enqueue(new Uint8Array(1024))
|
||||||
|
signal.addEventListener('abort', () => {
|
||||||
|
try { controller.error(signal.reason ?? new Error('aborted')) } catch { /* already errored */ }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return new globalThis.Response(stream, { status: 200 })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('downloadAssets abort handling', () => {
|
||||||
|
it('streams a zip and cleans up the staging dir on the happy path', async () => {
|
||||||
|
vi.stubGlobal('fetch', instantFetch(2048))
|
||||||
|
const res = new FakeRes()
|
||||||
|
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
|
||||||
|
expect(res.received).toBeGreaterThan(0)
|
||||||
|
expect(res.writableFinished).toBe(true)
|
||||||
|
expect(await stagingDirs()).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('resolves, cleans up and cancels upstream fetches when the client aborts mid-staging', async () => {
|
||||||
|
const signals: AbortSignal[] = []
|
||||||
|
vi.stubGlobal('fetch', trickleFetch(signals))
|
||||||
|
const res = new FakeRes()
|
||||||
|
const done = downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
|
||||||
|
// Give staging a moment to start, then drop the client connection
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 50))
|
||||||
|
res.destroy(new Error('client aborted'))
|
||||||
|
await done
|
||||||
|
expect(signals.length).toBeGreaterThan(0)
|
||||||
|
expect(signals.every(s => s.aborted)).toBe(true)
|
||||||
|
expect(await stagingDirs()).toEqual([])
|
||||||
|
}, 10_000)
|
||||||
|
|
||||||
|
it('resolves and cleans up when the client aborts while the zip is streaming', async () => {
|
||||||
|
// Large enough that archiver can't flush everything into a destroyed
|
||||||
|
// response; without disconnect handling finalize() would hang forever.
|
||||||
|
vi.stubGlobal('fetch', instantFetch(5 * 1024 * 1024))
|
||||||
|
const res = new FakeRes()
|
||||||
|
const origWrite = res._write.bind(res)
|
||||||
|
let first = true
|
||||||
|
res._write = (chunk, enc, cb) => {
|
||||||
|
origWrite(chunk, enc, cb)
|
||||||
|
if (first) {
|
||||||
|
first = false
|
||||||
|
setImmediate(() => res.destroy(new Error('client aborted')))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
|
||||||
|
expect(await stagingDirs()).toEqual([])
|
||||||
|
}, 10_000)
|
||||||
|
|
||||||
|
it('cleans up and destroys the response when an upstream fetch keeps failing', async () => {
|
||||||
|
vi.stubGlobal('fetch', vi.fn(async () => new globalThis.Response('nope', { status: 500 })))
|
||||||
|
const res = new FakeRes()
|
||||||
|
await downloadAssets(asResponse(res), share, [makeAsset('a1')])
|
||||||
|
expect(res.destroyed).toBe(true)
|
||||||
|
expect(await stagingDirs()).toEqual([])
|
||||||
|
}, 10_000)
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user