Fix #288 IPP reads from NAS at LAN speed and feeds user at a much slower speed, causing its buffer to overflow
This commit is contained in:
+52
-7
@@ -9,6 +9,9 @@ import { Asset, ImageSize, IncomingShareRequest, SharedLink } from '../types'
|
||||
import { respondToInvalidRequest } from '../invalidRequestHandler'
|
||||
import { getFilename } from '../gallery/filename'
|
||||
import { isVideoAsset, resolveDownloadEndpoint, resolveImageEndpoint } from '../gallery/sizing'
|
||||
import { pipeline } from 'stream/promises'
|
||||
import { readableFromWeb } from '../utils/webStream'
|
||||
import { log } from '../utils/log'
|
||||
|
||||
/**
|
||||
* Stream an asset from Immich back to the client.
|
||||
@@ -19,6 +22,15 @@ import { isVideoAsset, resolveDownloadEndpoint, resolveImageEndpoint } from '../
|
||||
* to serve them, and the upstream failure surfaces as a client 404.
|
||||
*/
|
||||
export async function assetBuffer (req: IncomingShareRequest, res: Response, asset: Asset, size?: ImageSize | string, share?: SharedLink, forceVideoPlayback = false) {
|
||||
/*
|
||||
Abort the upstream fetch as soon as the visitor goes away, so a cancelled
|
||||
download doesn't leave Immich streaming #288.
|
||||
*/
|
||||
const upstream = new AbortController()
|
||||
const onClose = () => { if (!res.writableFinished) upstream.abort() }
|
||||
res.once('close', onClose)
|
||||
if (res.closed) onClose()
|
||||
|
||||
const headerList = ['content-type', 'content-length', 'last-modified', 'etag']
|
||||
const fetchHeaders: Record<string, string> = {}
|
||||
let subpath: string
|
||||
@@ -66,7 +78,13 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
|
||||
|
||||
const url = assetFetchUrl(asset, subpath, sizeQueryParam)
|
||||
const reqHeaders = await authHeadersForAsset(asset)
|
||||
const data = await fetch(url, { headers: { ...fetchHeaders, ...reqHeaders } })
|
||||
let data: globalThis.Response
|
||||
try {
|
||||
data = await fetch(url, { headers: { ...fetchHeaders, ...reqHeaders }, signal: upstream.signal })
|
||||
} catch (e) {
|
||||
if (upstream.signal.aborted) return // visitor left before Immich answered
|
||||
throw e
|
||||
}
|
||||
|
||||
if (data.status < 200 || data.status >= 300) {
|
||||
let immichMessage = ''
|
||||
@@ -91,10 +109,37 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
|
||||
const value = data.headers.get(header)
|
||||
if (value) res.setHeader(header, value)
|
||||
})
|
||||
await data.body?.pipeTo(
|
||||
new WritableStream({
|
||||
write (chunk) { res.write(chunk) }
|
||||
})
|
||||
)
|
||||
res.end()
|
||||
|
||||
// Express routes HEAD through the GET handler and Node silently drops the
|
||||
// body writes, so without this we'd read the whole file from Immich for
|
||||
// nothing. The headers above are all a HEAD needs.
|
||||
if (req.req.method === 'HEAD' || !data.body) {
|
||||
await data.body?.cancel()
|
||||
res.end()
|
||||
return
|
||||
}
|
||||
|
||||
/*
|
||||
pipeline (rather than a WritableStream sink around res.write) honours
|
||||
backpressure from `res`: a LAN-speed read from Immich can't pile up in
|
||||
memory ahead of a slow visitor #288.
|
||||
*/
|
||||
try {
|
||||
await pipeline(readableFromWeb(data.body), res)
|
||||
} catch (e) {
|
||||
if (!isClientAbort(e)) {
|
||||
log.warn(`Stream from Immich failed for asset ${asset.id}: ${e instanceof Error ? e.message : String(e)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a pipeline error means the visitor went away: pipeline saw `res`
|
||||
* close early (or found it already closed), or our abort signal, fired from
|
||||
* that same close, reached the fetch body first.
|
||||
*/
|
||||
function isClientAbort (e: unknown): boolean {
|
||||
if (!(e instanceof Error)) return false
|
||||
const code = (e as NodeJS.ErrnoException).code
|
||||
return code === 'ERR_STREAM_PREMATURE_CLOSE' || code === 'ERR_STREAM_UNABLE_TO_PIPE' || e.name === 'AbortError'
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import { getNumericConfigOption } from '../config/access'
|
||||
import { log } from '../utils/log'
|
||||
import archiver, { Archiver } from 'archiver'
|
||||
import { sanitize } from '../utils/sanitize'
|
||||
import { Readable } from 'stream'
|
||||
import { pipeline } from 'stream/promises'
|
||||
import { promises as fs, createWriteStream } from 'fs'
|
||||
import { tmpdir } from 'os'
|
||||
@@ -15,6 +14,7 @@ import { title } from '../share'
|
||||
import { getFilename } from '../gallery/filename'
|
||||
import { createLimiter } from '../utils/limiter'
|
||||
import { createIdleTimeoutStream } from '../utils/idleTimeoutStream'
|
||||
import { readableFromWeb } from '../utils/webStream'
|
||||
|
||||
const STAGING_DIR_PREFIX = 'ipp-zip-'
|
||||
|
||||
@@ -341,12 +341,7 @@ async function fetchHeadersWithRetry (
|
||||
*/
|
||||
async function streamBodyToTempFile (response: globalThis.Response, tempfile: string, idleMs: number): Promise<{ ok: true } | { failure: unknown }> {
|
||||
if (!response.body) return { failure: new Error('Upstream response has no body') }
|
||||
// `response.body` is the global/undici ReadableStream<Uint8Array>;
|
||||
// Readable.fromWeb expects node:stream/web's ReadableStream<any>. The
|
||||
// two are structurally compatible at runtime but TS sees them as
|
||||
// distinct nominal types, so a cast is needed.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const body = Readable.fromWeb(response.body as any)
|
||||
const body = readableFromWeb(response.body)
|
||||
try {
|
||||
await pipeline(body, createIdleTimeoutStream(idleMs), createWriteStream(tempfile))
|
||||
return { ok: true }
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Readable } from 'stream'
|
||||
|
||||
/**
|
||||
* Wrap a fetch `Response.body` as a Node Readable, for use with `pipeline`.
|
||||
*
|
||||
* `response.body` is the global/undici ReadableStream<Uint8Array>;
|
||||
* Readable.fromWeb expects node:stream/web's ReadableStream<any>. The two
|
||||
* are structurally compatible at runtime but TS sees them as distinct
|
||||
* nominal types, so a cast is needed.
|
||||
*/
|
||||
export function readableFromWeb (body: ReadableStream<Uint8Array>): Readable {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return Readable.fromWeb(body as any)
|
||||
}
|
||||
Reference in New Issue
Block a user