Add nostore to upload healthcheck and fix double-sort

This commit is contained in:
Alan Grainger
2026-10-20 09:39:37 +13:00
parent fc7670157b
commit 6ae80569ff
10 changed files with 115 additions and 68 deletions
+21 -6
View File
@@ -5,7 +5,7 @@ import { resolveImageEndpoint } from './sizing'
/**
* Where an asset sits in its share, for the position-based download names of
* `downloadedFilename: 2`. Built with `positionInShare` / `findPositionInShare`.
* `downloadedFilename: 2`. Built with `positionInShare`, `sharePositions` or `findPositionInShare`.
*/
export interface AssetPosition {
/** The share's key. Only its hash reaches a filename, never the key itself. */
@@ -25,13 +25,28 @@ export function positionInShare (share: SharedLink, index: number): AssetPositio
}
/**
* Look up an asset's position in its share by id. A motion photo's clip takes
* the position of the still it belongs to. Undefined when the id is not in
* the share, which `getFilename` treats as "no position".
* Every asset's position in its share, by id, for a caller that looks up many.
* A motion photo's clip takes the position of the still it belongs to. The
* first asset to claim an id keeps it.
*/
export function sharePositions (share: SharedLink): Map<string, AssetPosition> {
const positions = new Map<string, AssetPosition>()
share.assets.forEach((asset, index) => {
const position = positionInShare(share, index)
for (const id of [asset.id, asset.livePhotoVideoId]) {
if (id && !positions.has(id)) positions.set(id, position)
}
})
return positions
}
/**
* Look up one asset's position in its share by id, as `sharePositions` does.
* Undefined when the id is not in the share, which `getFilename` treats as
* "no position".
*/
export function findPositionInShare (share: SharedLink, assetId: string): AssetPosition | undefined {
const index = share.assets.findIndex(a => a.id === assetId || a.livePhotoVideoId === assetId)
return index === -1 ? undefined : positionInShare(share, index)
return sharePositions(share).get(assetId)
}
/** `ipp.downloadedFilename` when the operator's config omits it: share hash plus position */
+17 -15
View File
@@ -6,36 +6,38 @@ import type { GroupByDateMode } from '../shared/types'
* when the share is loaded, so the gallery, the zip download and the single
* download all see the same order and an asset's position is its index.
*
* An album with a sort order is sorted by capture time in that direction.
* Date grouping then sorts by the local timestamp the client buckets on, so
* buckets stay contiguous (`dateSortComparator`).
* With date grouping on, sorts by the local timestamp the client buckets on,
* so buckets stay contiguous. Otherwise an album with a sort order is sorted
* by capture time in that direction, and anything else keeps Immich's order.
*
* Sorts in place and returns the same array.
*/
export function displayOrder (assets: Asset[], albumOrder?: string): Asset[] {
if (albumOrder === 'asc') {
assets.sort((a, b) => a?.fileCreatedAt?.localeCompare(b.fileCreatedAt || '') || 0)
} else if (albumOrder === 'desc') {
assets.sort((a, b) => b?.fileCreatedAt?.localeCompare(a.fileCreatedAt || '') || 0)
}
if (groupByDateMode()) {
assets.sort(dateSortComparator(albumOrder))
} else if (albumOrder === 'asc' || albumOrder === 'desc') {
assets.sort(dateSortComparator(albumOrder, captureTime))
}
return assets
}
/** The local timestamp the client's date groups bucket on */
const localTime = (a: Asset) => a.localDateTime || a.fileCreatedAt || ''
/** The capture timestamp in UTC, which Immich sorts an album by */
const captureTime = (a: Asset) => a.fileCreatedAt || ''
/**
* Comparator for the date-grouping sort: ascending when the album's order is
* `'asc'`, otherwise newest-first (individual shares and orderless albums).
* Comparator on the timestamp `key` returns: ascending when the album's order
* is `'asc'`, otherwise newest-first (individual shares and orderless albums).
* Undated assets always sort last regardless of direction, so the client's
* "Undated" group renders at the bottom.
* "Undated" group renders at the bottom and the order is consistent.
*/
export function dateSortComparator (order?: string): (a: Asset, b: Asset) => number {
export function dateSortComparator (order?: string, key: (a: Asset) => string = localTime): (a: Asset, b: Asset) => number {
const ascending = order === 'asc'
const sortKey = (a: Asset) => a.localDateTime || a.fileCreatedAt || ''
return (a, b) => {
const ka = sortKey(a)
const kb = sortKey(b)
const ka = key(a)
const kb = key(b)
if (!ka || !kb) return ka ? -1 : kb ? 1 : 0 // undated always last
return ascending ? ka.localeCompare(kb) : kb.localeCompare(ka)
}
+4 -4
View File
@@ -19,7 +19,7 @@ import { pipeline } from 'stream/promises'
* or locked assets are handled implicitly: Immich's own endpoints refuse
* to serve them, and the upstream failure surfaces as a client 404.
*/
export async function assetBuffer (req: IncomingShareRequest, res: Response, asset: Asset, size?: ImageSize | string, share?: SharedLink, forceVideoPlayback = false) {
export async function assetBuffer (req: IncomingShareRequest, res: Response, asset: Asset, size: ImageSize | string | undefined, share: SharedLink, forceVideoPlayback = false) {
// A cancelled download must not leave Immich streaming #288
const upstream = abortOnClose(res)
@@ -31,7 +31,7 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
let servedSize: ImageSize | undefined
const requested = validateImageSize(size)
const useVideoPlayback = forceVideoPlayback || (isVideoAsset(asset) && requested === ImageSize.original && share?.allowDownload === false)
const useVideoPlayback = forceVideoPlayback || (isVideoAsset(asset) && requested === ImageSize.original && share.allowDownload === false)
if (useVideoPlayback) {
subpath = '/video/playback'
@@ -60,7 +60,7 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
if (detail?.originalMimeType) asset = { ...asset, originalMimeType: detail.originalMimeType }
}
const endpoint = requested === ImageSize.original
? resolveDownloadEndpoint(asset, share?.allowDownload !== false)
? resolveDownloadEndpoint(asset, share.allowDownload !== false)
: resolveImageEndpoint(requested, asset)
subpath = endpoint.subpath
sizeQueryParam = endpoint.sizeQueryParam
@@ -95,7 +95,7 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
// Always an attachment, so an SVG can't render inline. Album assets have
// no originalFileName; Immich's own Content-Disposition carries it.
const named = asset.originalFileName ? asset : enrichFromHeaders(asset, data)
const position = share && findPositionInShare(share, asset.id)
const position = findPositionInShare(share, asset.id)
res.setHeader('Content-Disposition', attachmentDisposition(getFilename(named, servedSize, servedMimeFrom(subpath, data), position)))
}
headerList.forEach(header => {
+5 -5
View File
@@ -3,7 +3,7 @@ import { assetFetchUrl, authHeadersForAsset } from '../immich'
import { Response } from 'express-serve-static-core'
import archiver, { Archiver } from 'archiver'
import { resolveDownloadEndpoint, ImageEndpoint } from '../gallery/sizing'
import { AssetPosition, attachmentDisposition, enrichFromHeaders, findPositionInShare, getFilename, servedMimeFrom } from '../gallery/filename'
import { AssetPosition, attachmentDisposition, enrichFromHeaders, getFilename, servedMimeFrom, sharePositions } from '../gallery/filename'
/** Attempts to get response headers from Immich for one asset before giving up. */
const MAX_ATTEMPTS = 3
@@ -77,10 +77,12 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
// Headers and piping are deferred until the first asset has arrived, so a
// failure before then can still be answered with a normal error response.
let piped = false
// A multi-select download keeps each asset's position in the whole share
const positions = sharePositions(share)
for (const asset of assets) {
if (controller.signal.aborted) break
const fetched = await fetchOne(share, asset, controller.signal)
const fetched = await fetchOne(share, asset, positions.get(asset.id), controller.signal)
if (fetched === null) break // aborted while waiting on Immich
if ('failure' in fetched) {
if (!piped) {
@@ -196,7 +198,7 @@ function teardownArchive (archive: Archiver, res: Response) {
* Returns the fetched asset on success, a wrapped Failure on error, or null
* if the download was aborted before we got an answer.
*/
async function fetchOne (share: SharedLink, asset: Asset, signal: AbortSignal): Promise<FetchOutcome> {
async function fetchOne (share: SharedLink, asset: Asset, position: AssetPosition | undefined, signal: AbortSignal): Promise<FetchOutcome> {
if (signal.aborted) return null
const endpoint = resolveDownloadEndpoint(asset, share.allowDownload !== false)
@@ -213,8 +215,6 @@ async function fetchOne (share: SharedLink, asset: Asset, signal: AbortSignal):
// so recover them from the headers we already fetched - no extra calls.
const namedAsset = asset.originalFileName ? asset : enrichFromHeaders(asset, fetched.response)
// A multi-select download keeps each asset's position in the whole share
const position = findPositionInShare(share, asset.id)
return { response: fetched.response, asset: namedAsset, endpoint, servedMime: servedMimeFrom(endpoint.subpath, fetched.response), url, position }
}
+26 -12
View File
@@ -1,5 +1,5 @@
import { describe, it, expect, vi, afterEach } from 'vitest'
import { Asset, AssetType, KeyType, loadConfig } from '@ipp/core'
import { Asset, AssetType, KeyType, loadConfig, SharedLink } from '@ipp/core'
import { Writable } from 'stream'
import type { Request, Response } from 'express-serve-static-core'
import { assetBuffer } from '../src/stream/asset'
@@ -28,6 +28,11 @@ const asset: Asset = {
originalMimeType: 'image/jpeg'
}
/** A share holding just `item`, which every streamed asset belongs to. */
function shareOf (item: Asset): SharedLink {
return { key: 'testkey', keyType: KeyType.key, type: 'INDIVIDUAL', assets: [item] }
}
function makeRequest (method = 'GET'): IncomingShareRequest {
return { req: { method } as Request, key: 'testkey', range: '' }
}
@@ -117,10 +122,11 @@ function newSource (): Source {
describe('assetBuffer streaming', () => {
it('delivers the whole body and forwards upstream headers', async () => {
setConfig({ ipp: { downloadedFilename: 0 } })
const source = newSource()
vi.stubGlobal('fetch', pullFetch(4 * CHUNK, source))
const res = new FakeRes()
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
expect(res.received).toBe(4 * CHUNK)
expect(res.writableFinished).toBe(true)
expect(res.headers['content-type']).toBe('image/jpeg')
@@ -129,6 +135,13 @@ describe('assetBuffer streaming', () => {
expect(res.headers['x-accel-buffering']).toBe('no')
})
it('names a download by its position in the share by default', async () => {
vi.stubGlobal('fetch', pullFetch(CHUNK, newSource()))
const res = new FakeRes()
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
expect(res.headers['content-disposition']).toMatch(/''[0-9a-f]{8}_001\.jpg$/)
})
it('applies backpressure so upstream is read no faster than the visitor drains it', async () => {
// Issue #288: a fast NAS read against a slow visitor used to buffer the
// whole file in memory. Track how far ahead of the sink the source gets.
@@ -142,7 +155,7 @@ describe('assetBuffer streaming', () => {
maxOutstanding = Math.max(maxOutstanding, source.pulled - res.received)
origWrite(chunk, enc, cb)
}
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
expect(res.received).toBe(total)
expect(maxOutstanding).toBeLessThan(1024 * 1024)
}, 20_000)
@@ -160,7 +173,7 @@ describe('assetBuffer streaming', () => {
setImmediate(() => res.destroy(new Error('client aborted')))
}
}
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
expect(source.cancelled || source.signals[0].aborted).toBe(true)
// Upstream must stop shortly after the disconnect, not run to infinity
expect(source.pulled).toBeLessThan(1024 * 1024)
@@ -171,7 +184,7 @@ describe('assetBuffer streaming', () => {
vi.stubGlobal('fetch', pullFetch(Infinity, source, 200))
const res = new FakeRes()
setTimeout(() => res.destroy(new Error('client aborted')), 20)
await expect(assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)).resolves.toBeUndefined()
await expect(assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))).resolves.toBeUndefined()
expect(source.signals[0].aborted).toBe(true)
expect(source.pulled).toBe(0)
expect(res.received).toBe(0)
@@ -182,7 +195,7 @@ describe('assetBuffer streaming', () => {
vi.stubGlobal('fetch', pullFetch(Infinity, source, 50))
const res = new FakeRes()
res.destroy()
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
expect(source.signals[0].aborted).toBe(true)
expect(source.pulled).toBe(0)
})
@@ -191,7 +204,7 @@ describe('assetBuffer streaming', () => {
const source = newSource()
vi.stubGlobal('fetch', pullFetch(4 * CHUNK, source))
const res = new FakeRes()
await assetBuffer(makeRequest('HEAD'), asResponse(res), asset, ImageSize.original)
await assetBuffer(makeRequest('HEAD'), asResponse(res), asset, ImageSize.original, shareOf(asset))
expect(res.writableEnded).toBe(true)
expect(res.received).toBe(0)
expect(source.cancelled).toBe(true)
@@ -199,7 +212,7 @@ describe('assetBuffer streaming', () => {
})
it('names a WebP preview download .webp', async () => {
setConfig({ ipp: { maxDownloadQuality: 'preview' } })
setConfig({ ipp: { maxDownloadQuality: 'preview', downloadedFilename: 0 } })
vi.stubGlobal('fetch', vi.fn(async () => new globalThis.Response(
new ReadableStream({
start (controller) { controller.enqueue(new Uint8Array(1024)); controller.close() }
@@ -207,7 +220,7 @@ describe('assetBuffer streaming', () => {
{ status: 200, headers: { 'content-type': 'image/webp', 'content-length': '1024' } }
)))
const res = new FakeRes()
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
expect(res.headers['content-disposition']).toContain('a1.webp')
expect(res.headers['content-type']).toBe('image/webp')
})
@@ -227,15 +240,16 @@ describe('assetBuffer streaming', () => {
setConfig({ ipp: { downloadedFilename: 0 } })
vi.stubGlobal('fetch', svgFetch({ 'content-disposition': 'attachment; filename="drawing.svg"' }))
const res = new FakeRes()
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original)
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original, shareOf(albumAsset))
expect(res.headers['content-disposition']).toBe("attachment; filename*=UTF-8''drawing.svg")
expect(res.headers['x-content-type-options']).toBe('nosniff')
})
it('falls back to the asset id when Immich sends no filename', async () => {
setConfig({ ipp: { downloadedFilename: 0 } })
vi.stubGlobal('fetch', svgFetch({}))
const res = new FakeRes()
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original)
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original, shareOf(albumAsset))
expect(res.headers['content-disposition']).toBe("attachment; filename*=UTF-8''a2.svg")
})
})
@@ -243,7 +257,7 @@ describe('assetBuffer streaming', () => {
it('serves display sizes inline, still with nosniff', async () => {
vi.stubGlobal('fetch', pullFetch(CHUNK, newSource()))
const res = new FakeRes()
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.preview)
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.preview, shareOf(asset))
expect(res.headers['content-disposition']).toBeUndefined()
expect(res.headers['x-content-type-options']).toBe('nosniff')
})
+9 -3
View File
@@ -3,9 +3,9 @@ import { Asset, AssetType, KeyType, loadConfig } from '@ipp/core'
import { dateSortComparator, displayOrder } from '../src/gallery/order'
/*
The date-grouping sort must follow the album's own order while
keeping undated assets last in BOTH directions. displayOrder applies it,
after the album's own sort, when `gallery.groupByDate` is on.
The date sort must follow the album's own order while keeping undated
assets last in BOTH directions. displayOrder sorts once: by local time when
`gallery.groupByDate` is on, else by capture time in the album's order.
*/
afterEach(() => {
@@ -84,6 +84,12 @@ describe('displayOrder', () => {
expect(ids(displayOrder(album(), 'desc'))).toEqual(['b', 'a', 'c'])
})
it('puts undated assets last in an album sort', () => {
const withUndated = () => [dated('a', '2024-02-01T10:00:00.000Z'), { ...asset(), id: 'u' }, dated('c', '2024-01-01T10:00:00.000Z')]
expect(ids(displayOrder(withUndated(), 'asc'))).toEqual(['c', 'a', 'u'])
expect(ids(displayOrder(withUndated(), 'desc'))).toEqual(['a', 'c', 'u'])
})
it('sorts by local time when date grouping is on, so buckets stay contiguous', () => {
process.env.CONFIG = JSON.stringify({ ipp: { gallery: { groupByDate: 'day' } } })
loadConfig()