Fix #288 IPP reads from NAS at LAN speed and feeds user at a much slower speed, causing its buffer to overflow
This commit is contained in:
+51
-6
@@ -9,6 +9,9 @@ import { Asset, ImageSize, IncomingShareRequest, SharedLink } from '../types'
|
|||||||
import { respondToInvalidRequest } from '../invalidRequestHandler'
|
import { respondToInvalidRequest } from '../invalidRequestHandler'
|
||||||
import { getFilename } from '../gallery/filename'
|
import { getFilename } from '../gallery/filename'
|
||||||
import { isVideoAsset, resolveDownloadEndpoint, resolveImageEndpoint } from '../gallery/sizing'
|
import { isVideoAsset, resolveDownloadEndpoint, resolveImageEndpoint } from '../gallery/sizing'
|
||||||
|
import { pipeline } from 'stream/promises'
|
||||||
|
import { readableFromWeb } from '../utils/webStream'
|
||||||
|
import { log } from '../utils/log'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stream an asset from Immich back to the client.
|
* Stream an asset from Immich back to the client.
|
||||||
@@ -19,6 +22,15 @@ import { isVideoAsset, resolveDownloadEndpoint, resolveImageEndpoint } from '../
|
|||||||
* to serve them, and the upstream failure surfaces as a client 404.
|
* to serve them, and the upstream failure surfaces as a client 404.
|
||||||
*/
|
*/
|
||||||
export async function assetBuffer (req: IncomingShareRequest, res: Response, asset: Asset, size?: ImageSize | string, share?: SharedLink, forceVideoPlayback = false) {
|
export async function assetBuffer (req: IncomingShareRequest, res: Response, asset: Asset, size?: ImageSize | string, share?: SharedLink, forceVideoPlayback = false) {
|
||||||
|
/*
|
||||||
|
Abort the upstream fetch as soon as the visitor goes away, so a cancelled
|
||||||
|
download doesn't leave Immich streaming #288.
|
||||||
|
*/
|
||||||
|
const upstream = new AbortController()
|
||||||
|
const onClose = () => { if (!res.writableFinished) upstream.abort() }
|
||||||
|
res.once('close', onClose)
|
||||||
|
if (res.closed) onClose()
|
||||||
|
|
||||||
const headerList = ['content-type', 'content-length', 'last-modified', 'etag']
|
const headerList = ['content-type', 'content-length', 'last-modified', 'etag']
|
||||||
const fetchHeaders: Record<string, string> = {}
|
const fetchHeaders: Record<string, string> = {}
|
||||||
let subpath: string
|
let subpath: string
|
||||||
@@ -66,7 +78,13 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
|
|||||||
|
|
||||||
const url = assetFetchUrl(asset, subpath, sizeQueryParam)
|
const url = assetFetchUrl(asset, subpath, sizeQueryParam)
|
||||||
const reqHeaders = await authHeadersForAsset(asset)
|
const reqHeaders = await authHeadersForAsset(asset)
|
||||||
const data = await fetch(url, { headers: { ...fetchHeaders, ...reqHeaders } })
|
let data: globalThis.Response
|
||||||
|
try {
|
||||||
|
data = await fetch(url, { headers: { ...fetchHeaders, ...reqHeaders }, signal: upstream.signal })
|
||||||
|
} catch (e) {
|
||||||
|
if (upstream.signal.aborted) return // visitor left before Immich answered
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
|
||||||
if (data.status < 200 || data.status >= 300) {
|
if (data.status < 200 || data.status >= 300) {
|
||||||
let immichMessage = ''
|
let immichMessage = ''
|
||||||
@@ -91,10 +109,37 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
|
|||||||
const value = data.headers.get(header)
|
const value = data.headers.get(header)
|
||||||
if (value) res.setHeader(header, value)
|
if (value) res.setHeader(header, value)
|
||||||
})
|
})
|
||||||
await data.body?.pipeTo(
|
|
||||||
new WritableStream({
|
// Express routes HEAD through the GET handler and Node silently drops the
|
||||||
write (chunk) { res.write(chunk) }
|
// body writes, so without this we'd read the whole file from Immich for
|
||||||
})
|
// nothing. The headers above are all a HEAD needs.
|
||||||
)
|
if (req.req.method === 'HEAD' || !data.body) {
|
||||||
|
await data.body?.cancel()
|
||||||
res.end()
|
res.end()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
pipeline (rather than a WritableStream sink around res.write) honours
|
||||||
|
backpressure from `res`: a LAN-speed read from Immich can't pile up in
|
||||||
|
memory ahead of a slow visitor #288.
|
||||||
|
*/
|
||||||
|
try {
|
||||||
|
await pipeline(readableFromWeb(data.body), res)
|
||||||
|
} catch (e) {
|
||||||
|
if (!isClientAbort(e)) {
|
||||||
|
log.warn(`Stream from Immich failed for asset ${asset.id}: ${e instanceof Error ? e.message : String(e)}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* True when a pipeline error means the visitor went away: pipeline saw `res`
|
||||||
|
* close early (or found it already closed), or our abort signal, fired from
|
||||||
|
* that same close, reached the fetch body first.
|
||||||
|
*/
|
||||||
|
function isClientAbort (e: unknown): boolean {
|
||||||
|
if (!(e instanceof Error)) return false
|
||||||
|
const code = (e as NodeJS.ErrnoException).code
|
||||||
|
return code === 'ERR_STREAM_PREMATURE_CLOSE' || code === 'ERR_STREAM_UNABLE_TO_PIPE' || e.name === 'AbortError'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import { getNumericConfigOption } from '../config/access'
|
|||||||
import { log } from '../utils/log'
|
import { log } from '../utils/log'
|
||||||
import archiver, { Archiver } from 'archiver'
|
import archiver, { Archiver } from 'archiver'
|
||||||
import { sanitize } from '../utils/sanitize'
|
import { sanitize } from '../utils/sanitize'
|
||||||
import { Readable } from 'stream'
|
|
||||||
import { pipeline } from 'stream/promises'
|
import { pipeline } from 'stream/promises'
|
||||||
import { promises as fs, createWriteStream } from 'fs'
|
import { promises as fs, createWriteStream } from 'fs'
|
||||||
import { tmpdir } from 'os'
|
import { tmpdir } from 'os'
|
||||||
@@ -15,6 +14,7 @@ import { title } from '../share'
|
|||||||
import { getFilename } from '../gallery/filename'
|
import { getFilename } from '../gallery/filename'
|
||||||
import { createLimiter } from '../utils/limiter'
|
import { createLimiter } from '../utils/limiter'
|
||||||
import { createIdleTimeoutStream } from '../utils/idleTimeoutStream'
|
import { createIdleTimeoutStream } from '../utils/idleTimeoutStream'
|
||||||
|
import { readableFromWeb } from '../utils/webStream'
|
||||||
|
|
||||||
const STAGING_DIR_PREFIX = 'ipp-zip-'
|
const STAGING_DIR_PREFIX = 'ipp-zip-'
|
||||||
|
|
||||||
@@ -341,12 +341,7 @@ async function fetchHeadersWithRetry (
|
|||||||
*/
|
*/
|
||||||
async function streamBodyToTempFile (response: globalThis.Response, tempfile: string, idleMs: number): Promise<{ ok: true } | { failure: unknown }> {
|
async function streamBodyToTempFile (response: globalThis.Response, tempfile: string, idleMs: number): Promise<{ ok: true } | { failure: unknown }> {
|
||||||
if (!response.body) return { failure: new Error('Upstream response has no body') }
|
if (!response.body) return { failure: new Error('Upstream response has no body') }
|
||||||
// `response.body` is the global/undici ReadableStream<Uint8Array>;
|
const body = readableFromWeb(response.body)
|
||||||
// Readable.fromWeb expects node:stream/web's ReadableStream<any>. The
|
|
||||||
// two are structurally compatible at runtime but TS sees them as
|
|
||||||
// distinct nominal types, so a cast is needed.
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
||||||
const body = Readable.fromWeb(response.body as any)
|
|
||||||
try {
|
try {
|
||||||
await pipeline(body, createIdleTimeoutStream(idleMs), createWriteStream(tempfile))
|
await pipeline(body, createIdleTimeoutStream(idleMs), createWriteStream(tempfile))
|
||||||
return { ok: true }
|
return { ok: true }
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import { Readable } from 'stream'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrap a fetch `Response.body` as a Node Readable, for use with `pipeline`.
|
||||||
|
*
|
||||||
|
* `response.body` is the global/undici ReadableStream<Uint8Array>;
|
||||||
|
* Readable.fromWeb expects node:stream/web's ReadableStream<any>. The two
|
||||||
|
* are structurally compatible at runtime but TS sees them as distinct
|
||||||
|
* nominal types, so a cast is needed.
|
||||||
|
*/
|
||||||
|
export function readableFromWeb (body: ReadableStream<Uint8Array>): Readable {
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
return Readable.fromWeb(body as any)
|
||||||
|
}
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
import { describe, it, expect, vi, afterEach } from 'vitest'
|
||||||
|
import { Writable } from 'stream'
|
||||||
|
import type { Request, Response } from 'express-serve-static-core'
|
||||||
|
import { assetBuffer } from '../src/stream/asset'
|
||||||
|
import { Asset, AssetType, ImageSize, IncomingShareRequest, KeyType } from '../src/types'
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals()
|
||||||
|
})
|
||||||
|
|
||||||
|
const CHUNK = 64 * 1024
|
||||||
|
|
||||||
|
const asset: Asset = {
|
||||||
|
id: 'a1',
|
||||||
|
key: 'testkey',
|
||||||
|
keyType: KeyType.key,
|
||||||
|
type: AssetType.image,
|
||||||
|
isTrashed: false,
|
||||||
|
originalFileName: 'a1.jpg',
|
||||||
|
originalMimeType: 'image/jpeg'
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeRequest (method = 'GET'): IncomingShareRequest {
|
||||||
|
return { req: { method } as Request, key: 'testkey', range: '' }
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
Minimal stand-in for the Express response: a real Writable (so 'close',
|
||||||
|
writableFinished and destroy behave like the real thing) with a small
|
||||||
|
buffer and a sink that yields to the event loop on every chunk, like a
|
||||||
|
socket draining to a slow visitor.
|
||||||
|
*/
|
||||||
|
class FakeRes extends Writable {
|
||||||
|
received = 0
|
||||||
|
headers: Record<string, string> = {}
|
||||||
|
|
||||||
|
constructor () {
|
||||||
|
super({ highWaterMark: 16 * 1024 })
|
||||||
|
// Node's http stack handles response stream errors internally; a bare
|
||||||
|
// Writable would crash the test process on destroy(err) instead.
|
||||||
|
this.on('error', () => { /* swallowed, like http.ServerResponse */ })
|
||||||
|
}
|
||||||
|
|
||||||
|
setHeader (name: string, value: string) { this.headers[name.toLowerCase()] = value; return this }
|
||||||
|
status (_code: number) { return this }
|
||||||
|
|
||||||
|
_write (chunk: Buffer, _enc: string, cb: (error?: Error | null) => void) {
|
||||||
|
this.received += chunk.length
|
||||||
|
setImmediate(cb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function asResponse (res: FakeRes): Response {
|
||||||
|
return res as unknown as Response
|
||||||
|
}
|
||||||
|
|
||||||
|
type Source = {
|
||||||
|
pulled: number
|
||||||
|
cancelled: boolean
|
||||||
|
signals: AbortSignal[]
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch stub whose body is a pull-based stream: it hands out a chunk every
|
||||||
|
* time the consumer asks and never ahead of that, so `pulled` measures how
|
||||||
|
* far upstream has been read. Records cancellation and the request signal.
|
||||||
|
* `totalBytes` Infinity streams forever until cancelled.
|
||||||
|
*/
|
||||||
|
function pullFetch (totalBytes: number, source: Source, headersDelayMs = 0) {
|
||||||
|
return vi.fn(async (_url: unknown, init?: RequestInit) => {
|
||||||
|
const signal = init?.signal as AbortSignal
|
||||||
|
source.signals.push(signal)
|
||||||
|
const abortError = () => new DOMException('This operation was aborted', 'AbortError')
|
||||||
|
// Like real fetch: reject at once on an already-aborted signal, reject
|
||||||
|
// during the header wait, and error the body stream after that.
|
||||||
|
if (signal.aborted) throw abortError()
|
||||||
|
if (headersDelayMs) {
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(resolve, headersDelayMs)
|
||||||
|
signal.addEventListener('abort', () => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
reject(abortError())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const stream = new ReadableStream<Uint8Array>({
|
||||||
|
start (controller) {
|
||||||
|
signal.addEventListener('abort', () => {
|
||||||
|
try { controller.error(abortError()) } catch { /* already closed */ }
|
||||||
|
})
|
||||||
|
},
|
||||||
|
pull (controller) {
|
||||||
|
if (source.pulled >= totalBytes) return controller.close()
|
||||||
|
controller.enqueue(new Uint8Array(CHUNK))
|
||||||
|
source.pulled += CHUNK
|
||||||
|
},
|
||||||
|
cancel () { source.cancelled = true }
|
||||||
|
})
|
||||||
|
return new globalThis.Response(stream, {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'content-type': 'image/jpeg', 'content-length': String(totalBytes) }
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function newSource (): Source {
|
||||||
|
return { pulled: 0, cancelled: false, signals: [] }
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('assetBuffer streaming', () => {
|
||||||
|
it('delivers the whole body and forwards upstream headers', async () => {
|
||||||
|
const source = newSource()
|
||||||
|
vi.stubGlobal('fetch', pullFetch(4 * CHUNK, source))
|
||||||
|
const res = new FakeRes()
|
||||||
|
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||||
|
expect(res.received).toBe(4 * CHUNK)
|
||||||
|
expect(res.writableFinished).toBe(true)
|
||||||
|
expect(res.headers['content-type']).toBe('image/jpeg')
|
||||||
|
expect(res.headers['content-length']).toBe(String(4 * CHUNK))
|
||||||
|
expect(res.headers['content-disposition']).toContain('a1.jpg')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('applies backpressure so upstream is read no faster than the visitor drains it', async () => {
|
||||||
|
// Issue #288: a fast NAS read against a slow visitor used to buffer the
|
||||||
|
// whole file in memory. Track how far ahead of the sink the source gets.
|
||||||
|
const source = newSource()
|
||||||
|
const total = 20 * 1024 * 1024
|
||||||
|
vi.stubGlobal('fetch', pullFetch(total, source))
|
||||||
|
const res = new FakeRes()
|
||||||
|
let maxOutstanding = 0
|
||||||
|
const origWrite = res._write.bind(res)
|
||||||
|
res._write = (chunk, enc, cb) => {
|
||||||
|
maxOutstanding = Math.max(maxOutstanding, source.pulled - res.received)
|
||||||
|
origWrite(chunk, enc, cb)
|
||||||
|
}
|
||||||
|
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||||
|
expect(res.received).toBe(total)
|
||||||
|
expect(maxOutstanding).toBeLessThan(1024 * 1024)
|
||||||
|
}, 20_000)
|
||||||
|
|
||||||
|
it('cancels the upstream body when the visitor disconnects mid-stream', async () => {
|
||||||
|
const source = newSource()
|
||||||
|
vi.stubGlobal('fetch', pullFetch(Infinity, source))
|
||||||
|
const res = new FakeRes()
|
||||||
|
const origWrite = res._write.bind(res)
|
||||||
|
let first = true
|
||||||
|
res._write = (chunk, enc, cb) => {
|
||||||
|
origWrite(chunk, enc, cb)
|
||||||
|
if (first) {
|
||||||
|
first = false
|
||||||
|
setImmediate(() => res.destroy(new Error('client aborted')))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||||
|
expect(source.cancelled || source.signals[0].aborted).toBe(true)
|
||||||
|
// Upstream must stop shortly after the disconnect, not run to infinity
|
||||||
|
expect(source.pulled).toBeLessThan(1024 * 1024)
|
||||||
|
}, 10_000)
|
||||||
|
|
||||||
|
it('aborts the fetch when the visitor disconnects before Immich has answered', async () => {
|
||||||
|
const source = newSource()
|
||||||
|
vi.stubGlobal('fetch', pullFetch(Infinity, source, 200))
|
||||||
|
const res = new FakeRes()
|
||||||
|
setTimeout(() => res.destroy(new Error('client aborted')), 20)
|
||||||
|
await expect(assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)).resolves.toBeUndefined()
|
||||||
|
expect(source.signals[0].aborted).toBe(true)
|
||||||
|
expect(source.pulled).toBe(0)
|
||||||
|
expect(res.received).toBe(0)
|
||||||
|
}, 10_000)
|
||||||
|
|
||||||
|
it('does not fetch a body it will never write when the visitor left before we were called', async () => {
|
||||||
|
const source = newSource()
|
||||||
|
vi.stubGlobal('fetch', pullFetch(Infinity, source, 50))
|
||||||
|
const res = new FakeRes()
|
||||||
|
res.destroy()
|
||||||
|
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||||
|
expect(source.signals[0].aborted).toBe(true)
|
||||||
|
expect(source.pulled).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('answers HEAD with headers only and cancels the upstream body', async () => {
|
||||||
|
const source = newSource()
|
||||||
|
vi.stubGlobal('fetch', pullFetch(4 * CHUNK, source))
|
||||||
|
const res = new FakeRes()
|
||||||
|
await assetBuffer(makeRequest('HEAD'), asResponse(res), asset, ImageSize.original)
|
||||||
|
expect(res.writableEnded).toBe(true)
|
||||||
|
expect(res.received).toBe(0)
|
||||||
|
expect(source.cancelled).toBe(true)
|
||||||
|
expect(res.headers['content-length']).toBe(String(4 * CHUNK))
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -48,6 +48,7 @@ export default defineConfig({
|
|||||||
text: 'Deployment',
|
text: 'Deployment',
|
||||||
items: [
|
items: [
|
||||||
{ text: 'Single domain with Immich', link: '/running-on-single-domain' },
|
{ text: 'Single domain with Immich', link: '/running-on-single-domain' },
|
||||||
|
{ text: 'Redirect root domain to a share', link: '/redirect-root-to-share' },
|
||||||
{ text: 'Securing Immich with mTLS', link: '/securing-immich-with-mtls' },
|
{ text: 'Securing Immich with mTLS', link: '/securing-immich-with-mtls' },
|
||||||
{ text: 'Install with Kubernetes', link: '/kubernetes' }
|
{ text: 'Install with Kubernetes', link: '/kubernetes' }
|
||||||
]
|
]
|
||||||
|
|||||||
Reference in New Issue
Block a user