#289 stream downloads rather than staging to temp disk

This commit is contained in:
Alan Grainger
2026-09-04 11:20:45 +12:00
parent 02b30e0176
commit 9c72829c4f
7 changed files with 428 additions and 446 deletions
-192
View File
@@ -1,192 +0,0 @@
import { describe, it, expect, vi, beforeAll, afterAll, afterEach } from 'vitest'
import { promises as fs } from 'fs'
import { Writable } from 'stream'
import type { Response } from 'express-serve-static-core'
import { downloadAssets } from '../src/stream/download'
import { Asset, AssetType, KeyType, SharedLink } from '../src/types'
/*
Isolate this suite's staging dirs in a dedicated tmp dir so the
before/after assertions can't race other test files that also create
ipp-zip-* dirs under the real os.tmpdir().
*/
const { TEST_TMP } = vi.hoisted(() => {
return { TEST_TMP: (process.env.TMPDIR || '/tmp') + '/ipp-download-abort-test-' + process.pid }
})
vi.mock('os', async importOriginal => {
const orig = await importOriginal<typeof import('os')>()
return { ...orig, tmpdir: () => TEST_TMP }
})
beforeAll(async () => {
await fs.mkdir(TEST_TMP, { recursive: true })
})
afterAll(async () => {
await fs.rm(TEST_TMP, { recursive: true, force: true })
})
afterEach(() => {
vi.unstubAllGlobals()
})
async function stagingDirs (): Promise<string[]> {
const entries = await fs.readdir(TEST_TMP).catch(() => [] as string[])
return entries.filter(name => name.startsWith('ipp-zip-'))
}
/**
* Open fds pointing at deleted staging tempfiles. An aborted archiver used to
* keep the in-flight entry's read stream open forever; on tmpfs that pins the
* deleted file's space (issue #284). Linux-only introspection - returns []
* elsewhere, making the assertions no-ops.
*/
async function deletedStagingFds (): Promise<string[]> {
const fds = await fs.readdir('/proc/self/fd').catch(() => [] as string[])
const leaked: string[] = []
for (const fd of fds) {
const target = await fs.readlink('/proc/self/fd/' + fd).catch(() => '')
if (target.includes('ipp-zip-') && target.includes('(deleted)')) leaked.push(target)
}
return leaked
}
/** Wait for archiver's post-abort drain to release fds, then return leftovers. */
async function settledDeletedStagingFds (timeoutMs = 3000): Promise<string[]> {
const deadline = Date.now() + timeoutMs
let leaked = await deletedStagingFds()
while (leaked.length > 0 && Date.now() < deadline) {
await new Promise(resolve => setTimeout(resolve, 100))
leaked = await deletedStagingFds()
}
return leaked
}
function makeAsset (id: string): Asset {
return {
id,
key: 'testkey',
keyType: KeyType.key,
type: AssetType.image,
isTrashed: false,
originalFileName: id + '.jpg',
originalMimeType: 'image/jpeg'
}
}
const share: SharedLink = {
key: 'testkey',
keyType: KeyType.key,
type: 'ALBUM',
description: 'Test album',
assets: []
}
/*
Minimal stand-in for the Express response: a real Writable (so 'close',
writableFinished and destroy behave like the real thing) plus the header
setters downloadAssets calls.
*/
class FakeRes extends Writable {
received = 0
constructor () {
super()
// Node's http stack handles response stream errors internally; a bare
// Writable would crash the test process on destroy(err) instead.
this.on('error', () => { /* swallowed, like http.ServerResponse */ })
}
setHeader (_name: string, _value: string) { return this }
_write (chunk: Buffer, _enc: string, cb: (error?: Error | null) => void) {
this.received += chunk.length
cb()
}
}
function asResponse (res: FakeRes): Response {
return res as unknown as Response
}
/** Fetch stub that returns the full body immediately. */
function instantFetch (bytes: number) {
return vi.fn(async () => new globalThis.Response(new Uint8Array(bytes), { status: 200 }))
}
/**
* Fetch stub whose body sends one chunk then stays open forever, erroring
* only when the passed-in signal aborts - like a real streaming download
* that gets cancelled. Records each request's signal for assertions.
*/
function trickleFetch (signals: AbortSignal[]) {
return vi.fn(async (_url: unknown, init?: RequestInit) => {
const signal = init?.signal as AbortSignal
signals.push(signal)
const stream = new ReadableStream({
start (controller) {
controller.enqueue(new Uint8Array(1024))
signal.addEventListener('abort', () => {
try { controller.error(signal.reason ?? new Error('aborted')) } catch { /* already errored */ }
})
}
})
return new globalThis.Response(stream, { status: 200 })
})
}
describe('downloadAssets abort handling', () => {
it('streams a zip and cleans up the staging dir on the happy path', async () => {
vi.stubGlobal('fetch', instantFetch(2048))
const res = new FakeRes()
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
expect(res.received).toBeGreaterThan(0)
expect(res.writableFinished).toBe(true)
expect(await stagingDirs()).toEqual([])
})
it('resolves, cleans up and cancels upstream fetches when the client aborts mid-staging', async () => {
const signals: AbortSignal[] = []
vi.stubGlobal('fetch', trickleFetch(signals))
const res = new FakeRes()
const done = downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
// Give staging a moment to start, then drop the client connection
await new Promise(resolve => setTimeout(resolve, 50))
res.destroy(new Error('client aborted'))
await done
expect(signals.length).toBeGreaterThan(0)
expect(signals.every(s => s.aborted)).toBe(true)
expect(await stagingDirs()).toEqual([])
expect(await settledDeletedStagingFds()).toEqual([])
}, 10_000)
it('resolves and cleans up when the client aborts while the zip is streaming', async () => {
// Large enough that archiver can't flush everything into a destroyed
// response; without disconnect handling finalize() would hang forever.
vi.stubGlobal('fetch', instantFetch(5 * 1024 * 1024))
const res = new FakeRes()
const origWrite = res._write.bind(res)
let first = true
res._write = (chunk, enc, cb) => {
origWrite(chunk, enc, cb)
if (first) {
first = false
setImmediate(() => res.destroy(new Error('client aborted')))
}
}
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
expect(await stagingDirs()).toEqual([])
// The regression from issue #284's follow-up: the staging dir was removed
// but archiver still held an open fd on a deleted tempfile.
expect(await settledDeletedStagingFds()).toEqual([])
}, 10_000)
it('cleans up and destroys the response when an upstream fetch keeps failing', async () => {
vi.stubGlobal('fetch', vi.fn(async () => new globalThis.Response('nope', { status: 500 })))
const res = new FakeRes()
await downloadAssets(asResponse(res), share, [makeAsset('a1')])
expect(res.destroyed).toBe(true)
expect(await stagingDirs()).toEqual([])
expect(await settledDeletedStagingFds()).toEqual([])
}, 10_000)
})
+276
View File
@@ -0,0 +1,276 @@
import { describe, it, expect, vi, afterEach } from 'vitest'
import { Writable } from 'stream'
import { once } from 'events'
import type { Response } from 'express-serve-static-core'
import { downloadAssets } from '../src/stream/download'
import { Asset, AssetType, KeyType, SharedLink } from '../src/types'
/*
"Download all" streams each asset from Immich straight into the zip on the
wire - no disk staging, no buffering of whole files (issue #289). These tests
cover the shape of that pipeline: backpressure, one upstream body at a time,
and the three ways it can end early (visitor leaves, Immich refuses, body
dies mid-stream).
*/
afterEach(() => {
vi.unstubAllGlobals()
vi.restoreAllMocks()
})
function makeAsset (id: string): Asset {
return {
id,
key: 'testkey',
keyType: KeyType.key,
type: AssetType.image,
isTrashed: false,
originalFileName: id + '.jpg',
originalMimeType: 'image/jpeg'
}
}
const share: SharedLink = {
key: 'testkey',
keyType: KeyType.key,
type: 'ALBUM',
description: 'Test album',
assets: []
}
/*
Minimal stand-in for the Express response: a real Writable (so 'close',
writableFinished and destroy behave like the real thing) plus the header
and status helpers downloadAssets and respondToInvalidRequest call.
*/
class FakeRes extends Writable {
received = 0
chunks: Buffer[] = []
statusCode = 200
headers: Record<string, string> = {}
/** Delay applied to every write, to simulate a slow visitor. */
writeDelayMs = 0
constructor () {
super()
// Node's http stack handles response stream errors internally; a bare
// Writable would crash the test process on destroy(err) instead.
this.on('error', () => { /* swallowed, like http.ServerResponse */ })
}
setHeader (name: string, value: string) { this.headers[name] = value; return this }
status (code: number) { this.statusCode = code; return this }
send () { this.end(); return this }
_write (chunk: Buffer, _enc: string, cb: (error?: Error | null) => void) {
this.received += chunk.length
this.chunks.push(chunk)
if (this.writeDelayMs > 0) setTimeout(cb, this.writeDelayMs)
else cb()
}
get output () { return Buffer.concat(this.chunks) }
}
function asResponse (res: FakeRes): Response {
return res as unknown as Response
}
/** Fetch stub that returns the full body immediately. */
function instantFetch (bytes: number) {
return vi.fn(async () => new globalThis.Response(new Uint8Array(bytes), { status: 200 }))
}
/**
* Fetch stub whose body sends one chunk then stays open forever, erroring
* only when the passed-in signal aborts - like a real streaming download
* that gets cancelled. Records each request's signal for assertions.
*/
function trickleFetch (signals: AbortSignal[]) {
return vi.fn(async (_url: unknown, init?: RequestInit) => {
const signal = init?.signal as AbortSignal
signals.push(signal)
const stream = new ReadableStream({
start (controller) {
controller.enqueue(new Uint8Array(1024))
signal.addEventListener('abort', () => {
try { controller.error(signal.reason ?? new Error('aborted')) } catch { /* already errored */ }
})
}
})
return new globalThis.Response(stream, { status: 200 })
})
}
/**
* Fetch stub with a pull-based body: chunks are only produced on demand, and
* `pulled` counts how many bytes have been handed over so far.
*/
function pullFetch (totalBytes: number, chunkBytes: number, counter: { pulled: number }) {
return vi.fn(async () => {
let sent = 0
const stream = new ReadableStream({
pull (controller) {
if (sent >= totalBytes) {
controller.close()
return
}
const size = Math.min(chunkBytes, totalBytes - sent)
controller.enqueue(new Uint8Array(size))
sent += size
counter.pulled = sent
}
})
return new globalThis.Response(stream, { status: 200 })
})
}
/**
* Parse the central directory of a zip buffer: one {name, size} per entry.
* Just enough of the format to check the output is a real zip with the
* expected entries; extractors read the central directory, so it's the part
* that matters.
*/
function centralDirectory (zip: Buffer): Array<{ name: string, size: number }> {
const entries: Array<{ name: string, size: number }> = []
let offset = zip.indexOf('PK\x01\x02', 0, 'binary')
while (offset !== -1) {
const size = zip.readUInt32LE(offset + 24)
const nameLength = zip.readUInt16LE(offset + 28)
const name = zip.subarray(offset + 46, offset + 46 + nameLength).toString('utf8')
entries.push({ name, size })
offset = zip.indexOf('PK\x01\x02', offset + 46 + nameLength, 'binary')
}
return entries
}
function countOf (zip: Buffer, signature: string): number {
let count = 0
let offset = zip.indexOf(signature, 0, 'binary')
while (offset !== -1) {
count++
offset = zip.indexOf(signature, offset + 1, 'binary')
}
return count
}
describe('downloadAssets', () => {
it('streams a valid zip with one entry per asset', async () => {
vi.stubGlobal('fetch', instantFetch(2048))
const res = new FakeRes()
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
expect(res.writableFinished).toBe(true)
expect(res.headers['Content-Type']).toBe('application/zip')
const zip = res.output
expect(countOf(zip, 'PK\x03\x04')).toBe(2) // local file headers
expect(countOf(zip, 'PK\x05\x06')).toBe(1) // end of central directory
expect(centralDirectory(zip)).toEqual([
{ name: 'a1.jpg', size: 2048 },
{ name: 'a2.jpg', size: 2048 }
])
})
it('honours backpressure from a slow visitor instead of buffering the body', async () => {
const total = 32 * 1024 * 1024
const counter = { pulled: 0 }
vi.stubGlobal('fetch', pullFetch(total, 64 * 1024, counter))
const res = new FakeRes()
res.writeDelayMs = 1
let worstLead = 0
const origWrite = res._write.bind(res)
res._write = (chunk, enc, cb) => {
worstLead = Math.max(worstLead, counter.pulled - res.received)
origWrite(chunk, enc, cb)
}
await downloadAssets(asResponse(res), share, [makeAsset('a1')])
if (!res.writableFinished) await once(res, 'finish')
expect(counter.pulled).toBe(total)
// The lead is bounded by the stream buffers between the fetch body and
// the response: archiver and zip-stream each hold up to 1 MB per side,
// so roughly 4 MB in total. Without backpressure it would approach the
// whole 32 MB.
expect(worstLead).toBeLessThan(8 * 1024 * 1024)
}, 20_000)
it('fetches one asset at a time and cancels the upstream fetch when the client aborts mid-body', async () => {
const signals: AbortSignal[] = []
const fetchMock = trickleFetch(signals)
vi.stubGlobal('fetch', fetchMock)
const res = new FakeRes()
const done = downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2'), makeAsset('a3')])
// Give the first body a moment to start; nothing else should be in flight
await new Promise(resolve => setTimeout(resolve, 50))
expect(fetchMock).toHaveBeenCalledTimes(1)
res.destroy(new Error('client aborted'))
await done
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(signals.every(s => s.aborted)).toBe(true)
}, 10_000)
it('resolves when the client aborts while the zip is streaming', async () => {
// Large enough that archiver can't flush everything into a destroyed
// response; without disconnect handling finalize() would hang forever.
vi.stubGlobal('fetch', instantFetch(5 * 1024 * 1024))
const res = new FakeRes()
const origWrite = res._write.bind(res)
let first = true
res._write = (chunk, enc, cb) => {
origWrite(chunk, enc, cb)
if (first) {
first = false
setImmediate(() => res.destroy(new Error('client aborted')))
}
}
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
expect(res.destroyed).toBe(true)
}, 10_000)
it('answers 404 (not a broken zip) when the first asset fails before anything is sent', async () => {
vi.spyOn(console, 'log').mockImplementation(() => {})
vi.stubGlobal('fetch', vi.fn(async () => new globalThis.Response('nope', { status: 500 })))
const res = new FakeRes()
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
expect(res.statusCode).toBe(404)
expect(res.destroyed).toBe(false)
expect(res.writableEnded).toBe(true)
expect(res.received).toBe(0)
expect(res.headers['Content-Type']).toBeUndefined()
}, 10_000)
it('destroys the response when a later asset keeps failing', async () => {
vi.spyOn(console, 'log').mockImplementation(() => {})
let calls = 0
vi.stubGlobal('fetch', vi.fn(async () => {
calls++
return calls === 1
? new globalThis.Response(new Uint8Array(2048), { status: 200 })
: new globalThis.Response('nope', { status: 500 })
}))
const res = new FakeRes()
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2')])
expect(res.received).toBeGreaterThan(0)
expect(res.destroyed).toBe(true)
expect(res.writableFinished).toBe(false)
}, 10_000)
it('destroys the response and stops fetching when a body errors mid-stream', async () => {
vi.spyOn(console, 'log').mockImplementation(() => {})
const signals: AbortSignal[] = []
const fetchMock = vi.fn(async (_url: unknown, init?: RequestInit) => {
signals.push(init?.signal as AbortSignal)
const stream = new ReadableStream({
start (controller) {
controller.enqueue(new Uint8Array(1024))
setTimeout(() => controller.error(new Error('upstream died')), 20)
}
})
return new globalThis.Response(stream, { status: 200 })
})
vi.stubGlobal('fetch', fetchMock)
const res = new FakeRes()
await downloadAssets(asResponse(res), share, [makeAsset('a1'), makeAsset('a2'), makeAsset('a3')])
expect(res.destroyed).toBe(true)
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(signals.every(s => s.aborted)).toBe(true)
}, 10_000)
})
-63
View File
@@ -1,63 +0,0 @@
import { describe, it, expect, afterEach } from 'vitest'
import { promises as fs } from 'fs'
import { tmpdir } from 'os'
import { join } from 'path'
import { sweepStaleStagingDirs } from '../src/stream/download'
const PREFIX = 'ipp-zip-'
async function makeDir (name: string, mtimeMsAgo: number) {
const path = join(tmpdir(), name)
await fs.mkdir(path, { recursive: true })
await fs.writeFile(join(path, 'inside.bin'), 'x')
const mtime = new Date(Date.now() - mtimeMsAgo)
await fs.utimes(path, mtime, mtime)
return path
}
async function exists (path: string) {
try {
await fs.access(path)
return true
} catch {
return false
}
}
const created: string[] = []
afterEach(async () => {
await Promise.all(created.splice(0).map(p => fs.rm(p, { recursive: true, force: true }).catch(() => { /* best effort */ })))
})
describe('sweepStaleStagingDirs', () => {
it('deletes stale staging dirs older than the cutoff', async () => {
const stale = await makeDir(PREFIX + 'stale-' + process.pid + '-' + Math.random().toString(36).slice(2), 2 * 60 * 60 * 1000)
created.push(stale)
await sweepStaleStagingDirs(60 * 60 * 1000)
expect(await exists(stale)).toBe(false)
})
it('preserves staging dirs younger than the cutoff', async () => {
const fresh = await makeDir(PREFIX + 'fresh-' + process.pid + '-' + Math.random().toString(36).slice(2), 10 * 60 * 1000)
created.push(fresh)
await sweepStaleStagingDirs(60 * 60 * 1000)
expect(await exists(fresh)).toBe(true)
})
it('ignores directories that do not match the staging prefix', async () => {
const unrelated = await makeDir('not-ipp-zip-' + process.pid + '-' + Math.random().toString(36).slice(2), 24 * 60 * 60 * 1000)
created.push(unrelated)
await sweepStaleStagingDirs(60 * 60 * 1000)
expect(await exists(unrelated)).toBe(true)
})
it('uses its default 1h cutoff when called with no argument', async () => {
const stale = await makeDir(PREFIX + 'default-' + process.pid + '-' + Math.random().toString(36).slice(2), 2 * 60 * 60 * 1000)
const fresh = await makeDir(PREFIX + 'default2-' + process.pid + '-' + Math.random().toString(36).slice(2), 30 * 60 * 1000)
created.push(stale, fresh)
await sweepStaleStagingDirs()
expect(await exists(stale)).toBe(false)
expect(await exists(fresh)).toBe(true)
})
})