Add nostore to upload healthcheck and fix double-sort
This commit is contained in:
@@ -5,7 +5,7 @@ import { resolveImageEndpoint } from './sizing'
|
||||
|
||||
/**
|
||||
* Where an asset sits in its share, for the position-based download names of
|
||||
* `downloadedFilename: 2`. Built with `positionInShare` / `findPositionInShare`.
|
||||
* `downloadedFilename: 2`. Built with `positionInShare`, `sharePositions` or `findPositionInShare`.
|
||||
*/
|
||||
export interface AssetPosition {
|
||||
/** The share's key. Only its hash reaches a filename, never the key itself. */
|
||||
@@ -25,13 +25,28 @@ export function positionInShare (share: SharedLink, index: number): AssetPositio
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up an asset's position in its share by id. A motion photo's clip takes
|
||||
* the position of the still it belongs to. Undefined when the id is not in
|
||||
* the share, which `getFilename` treats as "no position".
|
||||
* Every asset's position in its share, by id, for a caller that looks up many.
|
||||
* A motion photo's clip takes the position of the still it belongs to. The
|
||||
* first asset to claim an id keeps it.
|
||||
*/
|
||||
export function sharePositions (share: SharedLink): Map<string, AssetPosition> {
|
||||
const positions = new Map<string, AssetPosition>()
|
||||
share.assets.forEach((asset, index) => {
|
||||
const position = positionInShare(share, index)
|
||||
for (const id of [asset.id, asset.livePhotoVideoId]) {
|
||||
if (id && !positions.has(id)) positions.set(id, position)
|
||||
}
|
||||
})
|
||||
return positions
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up one asset's position in its share by id, as `sharePositions` does.
|
||||
* Undefined when the id is not in the share, which `getFilename` treats as
|
||||
* "no position".
|
||||
*/
|
||||
export function findPositionInShare (share: SharedLink, assetId: string): AssetPosition | undefined {
|
||||
const index = share.assets.findIndex(a => a.id === assetId || a.livePhotoVideoId === assetId)
|
||||
return index === -1 ? undefined : positionInShare(share, index)
|
||||
return sharePositions(share).get(assetId)
|
||||
}
|
||||
|
||||
/** `ipp.downloadedFilename` when the operator's config omits it: share hash plus position */
|
||||
|
||||
+17
-15
@@ -6,36 +6,38 @@ import type { GroupByDateMode } from '../shared/types'
|
||||
* when the share is loaded, so the gallery, the zip download and the single
|
||||
* download all see the same order and an asset's position is its index.
|
||||
*
|
||||
* An album with a sort order is sorted by capture time in that direction.
|
||||
* Date grouping then sorts by the local timestamp the client buckets on, so
|
||||
* buckets stay contiguous (`dateSortComparator`).
|
||||
* With date grouping on, sorts by the local timestamp the client buckets on,
|
||||
* so buckets stay contiguous. Otherwise an album with a sort order is sorted
|
||||
* by capture time in that direction, and anything else keeps Immich's order.
|
||||
*
|
||||
* Sorts in place and returns the same array.
|
||||
*/
|
||||
export function displayOrder (assets: Asset[], albumOrder?: string): Asset[] {
|
||||
if (albumOrder === 'asc') {
|
||||
assets.sort((a, b) => a?.fileCreatedAt?.localeCompare(b.fileCreatedAt || '') || 0)
|
||||
} else if (albumOrder === 'desc') {
|
||||
assets.sort((a, b) => b?.fileCreatedAt?.localeCompare(a.fileCreatedAt || '') || 0)
|
||||
}
|
||||
if (groupByDateMode()) {
|
||||
assets.sort(dateSortComparator(albumOrder))
|
||||
} else if (albumOrder === 'asc' || albumOrder === 'desc') {
|
||||
assets.sort(dateSortComparator(albumOrder, captureTime))
|
||||
}
|
||||
return assets
|
||||
}
|
||||
|
||||
/** The local timestamp the client's date groups bucket on */
|
||||
const localTime = (a: Asset) => a.localDateTime || a.fileCreatedAt || ''
|
||||
|
||||
/** The capture timestamp in UTC, which Immich sorts an album by */
|
||||
const captureTime = (a: Asset) => a.fileCreatedAt || ''
|
||||
|
||||
/**
|
||||
* Comparator for the date-grouping sort: ascending when the album's order is
|
||||
* `'asc'`, otherwise newest-first (individual shares and orderless albums).
|
||||
* Comparator on the timestamp `key` returns: ascending when the album's order
|
||||
* is `'asc'`, otherwise newest-first (individual shares and orderless albums).
|
||||
* Undated assets always sort last regardless of direction, so the client's
|
||||
* "Undated" group renders at the bottom.
|
||||
* "Undated" group renders at the bottom and the order is consistent.
|
||||
*/
|
||||
export function dateSortComparator (order?: string): (a: Asset, b: Asset) => number {
|
||||
export function dateSortComparator (order?: string, key: (a: Asset) => string = localTime): (a: Asset, b: Asset) => number {
|
||||
const ascending = order === 'asc'
|
||||
const sortKey = (a: Asset) => a.localDateTime || a.fileCreatedAt || ''
|
||||
return (a, b) => {
|
||||
const ka = sortKey(a)
|
||||
const kb = sortKey(b)
|
||||
const ka = key(a)
|
||||
const kb = key(b)
|
||||
if (!ka || !kb) return ka ? -1 : kb ? 1 : 0 // undated always last
|
||||
return ascending ? ka.localeCompare(kb) : kb.localeCompare(ka)
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import { pipeline } from 'stream/promises'
|
||||
* or locked assets are handled implicitly: Immich's own endpoints refuse
|
||||
* to serve them, and the upstream failure surfaces as a client 404.
|
||||
*/
|
||||
export async function assetBuffer (req: IncomingShareRequest, res: Response, asset: Asset, size?: ImageSize | string, share?: SharedLink, forceVideoPlayback = false) {
|
||||
export async function assetBuffer (req: IncomingShareRequest, res: Response, asset: Asset, size: ImageSize | string | undefined, share: SharedLink, forceVideoPlayback = false) {
|
||||
// A cancelled download must not leave Immich streaming #288
|
||||
const upstream = abortOnClose(res)
|
||||
|
||||
@@ -31,7 +31,7 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
|
||||
let servedSize: ImageSize | undefined
|
||||
|
||||
const requested = validateImageSize(size)
|
||||
const useVideoPlayback = forceVideoPlayback || (isVideoAsset(asset) && requested === ImageSize.original && share?.allowDownload === false)
|
||||
const useVideoPlayback = forceVideoPlayback || (isVideoAsset(asset) && requested === ImageSize.original && share.allowDownload === false)
|
||||
|
||||
if (useVideoPlayback) {
|
||||
subpath = '/video/playback'
|
||||
@@ -60,7 +60,7 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
|
||||
if (detail?.originalMimeType) asset = { ...asset, originalMimeType: detail.originalMimeType }
|
||||
}
|
||||
const endpoint = requested === ImageSize.original
|
||||
? resolveDownloadEndpoint(asset, share?.allowDownload !== false)
|
||||
? resolveDownloadEndpoint(asset, share.allowDownload !== false)
|
||||
: resolveImageEndpoint(requested, asset)
|
||||
subpath = endpoint.subpath
|
||||
sizeQueryParam = endpoint.sizeQueryParam
|
||||
@@ -95,7 +95,7 @@ export async function assetBuffer (req: IncomingShareRequest, res: Response, ass
|
||||
// Always an attachment, so an SVG can't render inline. Album assets have
|
||||
// no originalFileName; Immich's own Content-Disposition carries it.
|
||||
const named = asset.originalFileName ? asset : enrichFromHeaders(asset, data)
|
||||
const position = share && findPositionInShare(share, asset.id)
|
||||
const position = findPositionInShare(share, asset.id)
|
||||
res.setHeader('Content-Disposition', attachmentDisposition(getFilename(named, servedSize, servedMimeFrom(subpath, data), position)))
|
||||
}
|
||||
headerList.forEach(header => {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { assetFetchUrl, authHeadersForAsset } from '../immich'
|
||||
import { Response } from 'express-serve-static-core'
|
||||
import archiver, { Archiver } from 'archiver'
|
||||
import { resolveDownloadEndpoint, ImageEndpoint } from '../gallery/sizing'
|
||||
import { AssetPosition, attachmentDisposition, enrichFromHeaders, findPositionInShare, getFilename, servedMimeFrom } from '../gallery/filename'
|
||||
import { AssetPosition, attachmentDisposition, enrichFromHeaders, getFilename, servedMimeFrom, sharePositions } from '../gallery/filename'
|
||||
|
||||
/** Attempts to get response headers from Immich for one asset before giving up. */
|
||||
const MAX_ATTEMPTS = 3
|
||||
@@ -77,10 +77,12 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
||||
// Headers and piping are deferred until the first asset has arrived, so a
|
||||
// failure before then can still be answered with a normal error response.
|
||||
let piped = false
|
||||
// A multi-select download keeps each asset's position in the whole share
|
||||
const positions = sharePositions(share)
|
||||
|
||||
for (const asset of assets) {
|
||||
if (controller.signal.aborted) break
|
||||
const fetched = await fetchOne(share, asset, controller.signal)
|
||||
const fetched = await fetchOne(share, asset, positions.get(asset.id), controller.signal)
|
||||
if (fetched === null) break // aborted while waiting on Immich
|
||||
if ('failure' in fetched) {
|
||||
if (!piped) {
|
||||
@@ -196,7 +198,7 @@ function teardownArchive (archive: Archiver, res: Response) {
|
||||
* Returns the fetched asset on success, a wrapped Failure on error, or null
|
||||
* if the download was aborted before we got an answer.
|
||||
*/
|
||||
async function fetchOne (share: SharedLink, asset: Asset, signal: AbortSignal): Promise<FetchOutcome> {
|
||||
async function fetchOne (share: SharedLink, asset: Asset, position: AssetPosition | undefined, signal: AbortSignal): Promise<FetchOutcome> {
|
||||
if (signal.aborted) return null
|
||||
|
||||
const endpoint = resolveDownloadEndpoint(asset, share.allowDownload !== false)
|
||||
@@ -213,8 +215,6 @@ async function fetchOne (share: SharedLink, asset: Asset, signal: AbortSignal):
|
||||
// so recover them from the headers we already fetched - no extra calls.
|
||||
const namedAsset = asset.originalFileName ? asset : enrichFromHeaders(asset, fetched.response)
|
||||
|
||||
// A multi-select download keeps each asset's position in the whole share
|
||||
const position = findPositionInShare(share, asset.id)
|
||||
return { response: fetched.response, asset: namedAsset, endpoint, servedMime: servedMimeFrom(endpoint.subpath, fetched.response), url, position }
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest'
|
||||
import { Asset, AssetType, KeyType, loadConfig } from '@ipp/core'
|
||||
import { Asset, AssetType, KeyType, loadConfig, SharedLink } from '@ipp/core'
|
||||
import { Writable } from 'stream'
|
||||
import type { Request, Response } from 'express-serve-static-core'
|
||||
import { assetBuffer } from '../src/stream/asset'
|
||||
@@ -28,6 +28,11 @@ const asset: Asset = {
|
||||
originalMimeType: 'image/jpeg'
|
||||
}
|
||||
|
||||
/** A share holding just `item`, which every streamed asset belongs to. */
|
||||
function shareOf (item: Asset): SharedLink {
|
||||
return { key: 'testkey', keyType: KeyType.key, type: 'INDIVIDUAL', assets: [item] }
|
||||
}
|
||||
|
||||
function makeRequest (method = 'GET'): IncomingShareRequest {
|
||||
return { req: { method } as Request, key: 'testkey', range: '' }
|
||||
}
|
||||
@@ -117,10 +122,11 @@ function newSource (): Source {
|
||||
|
||||
describe('assetBuffer streaming', () => {
|
||||
it('delivers the whole body and forwards upstream headers', async () => {
|
||||
setConfig({ ipp: { downloadedFilename: 0 } })
|
||||
const source = newSource()
|
||||
vi.stubGlobal('fetch', pullFetch(4 * CHUNK, source))
|
||||
const res = new FakeRes()
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
|
||||
expect(res.received).toBe(4 * CHUNK)
|
||||
expect(res.writableFinished).toBe(true)
|
||||
expect(res.headers['content-type']).toBe('image/jpeg')
|
||||
@@ -129,6 +135,13 @@ describe('assetBuffer streaming', () => {
|
||||
expect(res.headers['x-accel-buffering']).toBe('no')
|
||||
})
|
||||
|
||||
it('names a download by its position in the share by default', async () => {
|
||||
vi.stubGlobal('fetch', pullFetch(CHUNK, newSource()))
|
||||
const res = new FakeRes()
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
|
||||
expect(res.headers['content-disposition']).toMatch(/''[0-9a-f]{8}_001\.jpg$/)
|
||||
})
|
||||
|
||||
it('applies backpressure so upstream is read no faster than the visitor drains it', async () => {
|
||||
// Issue #288: a fast NAS read against a slow visitor used to buffer the
|
||||
// whole file in memory. Track how far ahead of the sink the source gets.
|
||||
@@ -142,7 +155,7 @@ describe('assetBuffer streaming', () => {
|
||||
maxOutstanding = Math.max(maxOutstanding, source.pulled - res.received)
|
||||
origWrite(chunk, enc, cb)
|
||||
}
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
|
||||
expect(res.received).toBe(total)
|
||||
expect(maxOutstanding).toBeLessThan(1024 * 1024)
|
||||
}, 20_000)
|
||||
@@ -160,7 +173,7 @@ describe('assetBuffer streaming', () => {
|
||||
setImmediate(() => res.destroy(new Error('client aborted')))
|
||||
}
|
||||
}
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
|
||||
expect(source.cancelled || source.signals[0].aborted).toBe(true)
|
||||
// Upstream must stop shortly after the disconnect, not run to infinity
|
||||
expect(source.pulled).toBeLessThan(1024 * 1024)
|
||||
@@ -171,7 +184,7 @@ describe('assetBuffer streaming', () => {
|
||||
vi.stubGlobal('fetch', pullFetch(Infinity, source, 200))
|
||||
const res = new FakeRes()
|
||||
setTimeout(() => res.destroy(new Error('client aborted')), 20)
|
||||
await expect(assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)).resolves.toBeUndefined()
|
||||
await expect(assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))).resolves.toBeUndefined()
|
||||
expect(source.signals[0].aborted).toBe(true)
|
||||
expect(source.pulled).toBe(0)
|
||||
expect(res.received).toBe(0)
|
||||
@@ -182,7 +195,7 @@ describe('assetBuffer streaming', () => {
|
||||
vi.stubGlobal('fetch', pullFetch(Infinity, source, 50))
|
||||
const res = new FakeRes()
|
||||
res.destroy()
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
|
||||
expect(source.signals[0].aborted).toBe(true)
|
||||
expect(source.pulled).toBe(0)
|
||||
})
|
||||
@@ -191,7 +204,7 @@ describe('assetBuffer streaming', () => {
|
||||
const source = newSource()
|
||||
vi.stubGlobal('fetch', pullFetch(4 * CHUNK, source))
|
||||
const res = new FakeRes()
|
||||
await assetBuffer(makeRequest('HEAD'), asResponse(res), asset, ImageSize.original)
|
||||
await assetBuffer(makeRequest('HEAD'), asResponse(res), asset, ImageSize.original, shareOf(asset))
|
||||
expect(res.writableEnded).toBe(true)
|
||||
expect(res.received).toBe(0)
|
||||
expect(source.cancelled).toBe(true)
|
||||
@@ -199,7 +212,7 @@ describe('assetBuffer streaming', () => {
|
||||
})
|
||||
|
||||
it('names a WebP preview download .webp', async () => {
|
||||
setConfig({ ipp: { maxDownloadQuality: 'preview' } })
|
||||
setConfig({ ipp: { maxDownloadQuality: 'preview', downloadedFilename: 0 } })
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new globalThis.Response(
|
||||
new ReadableStream({
|
||||
start (controller) { controller.enqueue(new Uint8Array(1024)); controller.close() }
|
||||
@@ -207,7 +220,7 @@ describe('assetBuffer streaming', () => {
|
||||
{ status: 200, headers: { 'content-type': 'image/webp', 'content-length': '1024' } }
|
||||
)))
|
||||
const res = new FakeRes()
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original)
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.original, shareOf(asset))
|
||||
expect(res.headers['content-disposition']).toContain('a1.webp')
|
||||
expect(res.headers['content-type']).toBe('image/webp')
|
||||
})
|
||||
@@ -227,15 +240,16 @@ describe('assetBuffer streaming', () => {
|
||||
setConfig({ ipp: { downloadedFilename: 0 } })
|
||||
vi.stubGlobal('fetch', svgFetch({ 'content-disposition': 'attachment; filename="drawing.svg"' }))
|
||||
const res = new FakeRes()
|
||||
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original)
|
||||
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original, shareOf(albumAsset))
|
||||
expect(res.headers['content-disposition']).toBe("attachment; filename*=UTF-8''drawing.svg")
|
||||
expect(res.headers['x-content-type-options']).toBe('nosniff')
|
||||
})
|
||||
|
||||
it('falls back to the asset id when Immich sends no filename', async () => {
|
||||
setConfig({ ipp: { downloadedFilename: 0 } })
|
||||
vi.stubGlobal('fetch', svgFetch({}))
|
||||
const res = new FakeRes()
|
||||
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original)
|
||||
await assetBuffer(makeRequest(), asResponse(res), albumAsset, ImageSize.original, shareOf(albumAsset))
|
||||
expect(res.headers['content-disposition']).toBe("attachment; filename*=UTF-8''a2.svg")
|
||||
})
|
||||
})
|
||||
@@ -243,7 +257,7 @@ describe('assetBuffer streaming', () => {
|
||||
it('serves display sizes inline, still with nosniff', async () => {
|
||||
vi.stubGlobal('fetch', pullFetch(CHUNK, newSource()))
|
||||
const res = new FakeRes()
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.preview)
|
||||
await assetBuffer(makeRequest(), asResponse(res), asset, ImageSize.preview, shareOf(asset))
|
||||
expect(res.headers['content-disposition']).toBeUndefined()
|
||||
expect(res.headers['x-content-type-options']).toBe('nosniff')
|
||||
})
|
||||
|
||||
@@ -3,9 +3,9 @@ import { Asset, AssetType, KeyType, loadConfig } from '@ipp/core'
|
||||
import { dateSortComparator, displayOrder } from '../src/gallery/order'
|
||||
|
||||
/*
|
||||
The date-grouping sort must follow the album's own order while
|
||||
keeping undated assets last in BOTH directions. displayOrder applies it,
|
||||
after the album's own sort, when `gallery.groupByDate` is on.
|
||||
The date sort must follow the album's own order while keeping undated
|
||||
assets last in BOTH directions. displayOrder sorts once: by local time when
|
||||
`gallery.groupByDate` is on, else by capture time in the album's order.
|
||||
*/
|
||||
|
||||
afterEach(() => {
|
||||
@@ -84,6 +84,12 @@ describe('displayOrder', () => {
|
||||
expect(ids(displayOrder(album(), 'desc'))).toEqual(['b', 'a', 'c'])
|
||||
})
|
||||
|
||||
it('puts undated assets last in an album sort', () => {
|
||||
const withUndated = () => [dated('a', '2024-02-01T10:00:00.000Z'), { ...asset(), id: 'u' }, dated('c', '2024-01-01T10:00:00.000Z')]
|
||||
expect(ids(displayOrder(withUndated(), 'asc'))).toEqual(['c', 'a', 'u'])
|
||||
expect(ids(displayOrder(withUndated(), 'desc'))).toEqual(['a', 'c', 'u'])
|
||||
})
|
||||
|
||||
it('sorts by local time when date grouping is on, so buckets stay contiguous', () => {
|
||||
process.env.CONFIG = JSON.stringify({ ipp: { gallery: { groupByDate: 'day' } } })
|
||||
loadConfig()
|
||||
|
||||
Reference in New Issue
Block a user