Merge pull request #218 from ahyattdev/limit-concurrent-download
Limit concurrent downloads and retry failed downloads
This commit is contained in:
@@ -159,18 +159,19 @@ Alternatively, you can [pass the configuration inline](docs/inline-configuration
|
|||||||
|
|
||||||
### Immich Public Proxy options
|
### Immich Public Proxy options
|
||||||
|
|
||||||
| Option | Type | Description |
|
| Option | Type | Description |
|
||||||
|--------------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|---------------------------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `responseHeaders` | `object` | Change the headers sent with your web responses. By default there is `cache-control` and CORS added. |
|
| `responseHeaders` | `object` | Change the headers sent with your web responses. By default there is `cache-control` and CORS added. |
|
||||||
| `downloadOriginalPhoto` | `bool` | Set to `false` if you only want people to be able to download the 'preview' quality photo, rather than your original photo. |
|
| `downloadOriginalPhoto` | `bool` | Set to `false` if you only want people to be able to download the 'preview' quality photo, rather than your original photo. |
|
||||||
| `downloadedFilename` | `int` | The filename of the downloaded image.<br>`0` for the original filename if available, falling back to the Immich asset ID<br>`1` for the Immich asset ID number<br>`2` for a shortened version of the asset ID: `img_` plus the first 8 characters of the asset ID |
|
| `downloadedFilename` | `int` | The filename of the downloaded image.<br>`0` for the original filename if available, falling back to the Immich asset ID<br>`1` for the Immich asset ID number<br>`2` for a shortened version of the asset ID: `img_` plus the first 8 characters of the asset ID |
|
||||||
| `allowDownloadAll` | `int` | Allow visitors to download all files as a zip, including the selective-download flow in multi-select mode.<br>`0` disable downloads<br>`1` follow Immich setting per share ([example](https://github.com/user-attachments/assets/79ea8c08-71ce-42ab-b025-10aec384938a))<br>`2` always allowed |
|
| `allowDownloadAll` | `int` | Allow visitors to download all files as a zip, including the selective-download flow in multi-select mode.<br>`0` disable downloads<br>`1` follow Immich setting per share ([example](https://github.com/user-attachments/assets/79ea8c08-71ce-42ab-b025-10aec384938a))<br>`2` always allowed |
|
||||||
| `allowSlugLinks` | `bool` | Enable/disable the custom URL links. |
|
| `downloadFromImmichConcurrencyLimit` | `int` | Maximum number of assets IPP will fetch from your Immich server in parallel when building a "download all" zip. Defaults to `20`. Lower this if your Immich server is slow or you see download timeouts on large albums; raise it for faster downloads if your server can handle the load. |
|
||||||
| `showHomePage` | `bool` | Set to `false` to remove the IPP shield page at `/` and at `/share` |
|
| `allowSlugLinks` | `bool` | Enable/disable the custom URL links. |
|
||||||
| `gallery` | `object` | Gallery-page options, see the [Gallery options](#gallery-options) section below. |
|
| `showHomePage` | `bool` | Set to `false` to remove the IPP shield page at `/` and at `/share` |
|
||||||
| `lightbox` | `object` | Lightbox option, see the [Lightbox options](#lightbox-options) section below. |
|
| `gallery` | `object` | Gallery-page options, see the [Gallery options](#gallery-options) section below. |
|
||||||
| `showMetadata` | `object` | See the [Metadata](#metadata) section below. |
|
| `lightbox` | `object` | Lightbox option, see the [Lightbox options](#lightbox-options) section below. |
|
||||||
| `customInvalidResponse` | various | Send a custom response instead of the default 404 - see [Custom responses](docs/custom-responses.md) for more details. |
|
| `showMetadata` | `object` | See the [Metadata](#metadata) section below. |
|
||||||
|
| `customInvalidResponse` | various | Send a custom response instead of the default 404 - see [Custom responses](docs/custom-responses.md) for more details. |
|
||||||
|
|
||||||
For example, to disable the home page at `/` and at `/share` you need to change `showHomePage` to `false`:
|
For example, to disable the home page at `/` and at `/share` you need to change `showHomePage` to `false`:
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
"downloadOriginalPhoto": true,
|
"downloadOriginalPhoto": true,
|
||||||
"downloadedFilename": 0,
|
"downloadedFilename": 0,
|
||||||
"allowDownloadAll": 0,
|
"allowDownloadAll": 0,
|
||||||
|
"downloadFromImmichConcurrencyLimit": 20,
|
||||||
"allowSlugLinks": true,
|
"allowSlugLinks": true,
|
||||||
"showHomePage": true,
|
"showHomePage": true,
|
||||||
"showMetadata": {
|
"showMetadata": {
|
||||||
|
|||||||
+88
-16
@@ -1,7 +1,7 @@
|
|||||||
import immich from './immich'
|
import immich from './immich'
|
||||||
import { Response } from 'express-serve-static-core'
|
import { Response } from 'express-serve-static-core'
|
||||||
import { Asset, AssetType, ImageSize, IncomingShareRequest, SharedLink } from './types'
|
import { Asset, AssetType, ImageSize, IncomingShareRequest, SharedLink } from './types'
|
||||||
import { canDownload, escapeHtml, getConfigOption, toString } from './functions'
|
import { canDownload, escapeHtml, getConfigOption, log, toString } from './functions'
|
||||||
import archiver from 'archiver'
|
import archiver from 'archiver'
|
||||||
import { respondToInvalidRequest } from './invalidRequestHandler'
|
import { respondToInvalidRequest } from './invalidRequestHandler'
|
||||||
import { sanitize } from './includes/sanitize'
|
import { sanitize } from './includes/sanitize'
|
||||||
@@ -202,7 +202,7 @@ class Render {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Download all assets in a share as a zip file
|
* Download all assets in a share as a zip file.
|
||||||
*/
|
*/
|
||||||
async downloadAll (res: Response, share: SharedLink) {
|
async downloadAll (res: Response, share: SharedLink) {
|
||||||
await this.downloadAssets(res, share, share.assets)
|
await this.downloadAssets(res, share, share.assets)
|
||||||
@@ -211,31 +211,103 @@ class Render {
|
|||||||
/**
|
/**
|
||||||
* Stream the given assets back as a zip file. Caller is responsible for
|
* Stream the given assets back as a zip file. Caller is responsible for
|
||||||
* checking that every asset belongs to the share (security boundary).
|
* checking that every asset belongs to the share (security boundary).
|
||||||
|
*
|
||||||
|
* Assets are fetched from Immich with a bounded concurrency (configurable via
|
||||||
|
* `ipp.downloadFromImmichConcurrencyLimit`, default 20) rather than all at once.
|
||||||
|
* Fetching every asset in parallel overwhelms the Immich server on large albums
|
||||||
|
* and causes the client download to time out before the first bytes are sent.
|
||||||
|
*
|
||||||
|
* Each asset is retried up to 3 times with linear backoff. If any asset still
|
||||||
|
* fails after retries, the in-flight download is aborted mid-stream: the
|
||||||
|
* archive is aborted, the HTTP response socket is destroyed, and the failure
|
||||||
|
* is logged. We do this (rather than silently omitting the asset) because by
|
||||||
|
* the time we know the asset failed we've already sent a 200 response to the
|
||||||
|
* client, so the only way to signal "your zip is incomplete" is to terminate
|
||||||
|
* the response — leaving the user with a visibly broken/truncated download
|
||||||
|
* instead of a zip that's quietly missing files.
|
||||||
*/
|
*/
|
||||||
async downloadAssets (res: Response, share: SharedLink, assets: Asset[]) {
|
async downloadAssets (res: Response, share: SharedLink, assets: Asset[]) {
|
||||||
const downloadOriginalAsset = getConfigOption('ipp.downloadOriginalPhoto', true)
|
const downloadOriginalAsset = getConfigOption('ipp.downloadOriginalPhoto', true)
|
||||||
|
const concurrency = Math.max(1, getConfigOption('ipp.downloadFromImmichConcurrencyLimit', 20) as number)
|
||||||
res.setHeader('Content-Type', 'application/zip')
|
res.setHeader('Content-Type', 'application/zip')
|
||||||
let filename = (sanitize(this.title(share)) || 'photos') + '.zip'
|
let filename = (sanitize(this.title(share)) || 'photos') + '.zip'
|
||||||
filename = encodeURI(filename)
|
filename = encodeURI(filename)
|
||||||
res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${filename}`)
|
res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${filename}`)
|
||||||
const archive = archiver('zip', { zlib: { level: 6 } })
|
const archive = archiver('zip', { zlib: { level: 6 } })
|
||||||
archive.pipe(res)
|
archive.pipe(res)
|
||||||
await Promise.all(assets.map(async (asset) => {
|
|
||||||
const endpoint = downloadOriginalAsset ? 'original' : 'thumbnail'
|
let aborted = false
|
||||||
const url = immich.buildUrl(immich.apiUrl() + '/assets/' + encodeURIComponent(asset.id) + '/' + endpoint, {
|
let failure: { asset: Asset, url: string, status?: number, error?: unknown } | null = null
|
||||||
key: asset.key,
|
const fail = (info: { asset: Asset, url: string, status?: number, error?: unknown }) => {
|
||||||
password: asset.password,
|
if (!aborted) {
|
||||||
size: downloadOriginalAsset ? '' : 'preview'
|
aborted = true
|
||||||
})
|
failure = info
|
||||||
const data = await fetch(url)
|
|
||||||
if (!data.ok) {
|
|
||||||
console.warn(`Failed to fetch asset: ${asset.id}`)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
archive.append(Buffer.from(await data.arrayBuffer()), { name: this.getFilename(asset) })
|
}
|
||||||
}))
|
|
||||||
|
const maxAttempts = 3
|
||||||
|
const queue = assets.slice()
|
||||||
|
const worker = async () => {
|
||||||
|
while (queue.length > 0) {
|
||||||
|
if (aborted) return
|
||||||
|
const asset = queue.shift()
|
||||||
|
if (!asset) return
|
||||||
|
const endpoint = downloadOriginalAsset ? 'original' : 'thumbnail'
|
||||||
|
const url = immich.buildUrl(immich.apiUrl() + '/assets/' + encodeURIComponent(asset.id) + '/' + endpoint, {
|
||||||
|
key: asset.key,
|
||||||
|
password: asset.password,
|
||||||
|
size: downloadOriginalAsset ? '' : 'preview'
|
||||||
|
})
|
||||||
|
|
||||||
|
let buffer: Buffer | undefined
|
||||||
|
let lastStatus: number | undefined
|
||||||
|
let lastError: unknown
|
||||||
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
|
if (aborted) return
|
||||||
|
try {
|
||||||
|
const data = await fetch(url)
|
||||||
|
if (data.ok) {
|
||||||
|
buffer = Buffer.from(await data.arrayBuffer())
|
||||||
|
break
|
||||||
|
}
|
||||||
|
lastStatus = data.status
|
||||||
|
lastError = undefined
|
||||||
|
} catch (e) {
|
||||||
|
lastError = e
|
||||||
|
lastStatus = undefined
|
||||||
|
}
|
||||||
|
if (attempt < maxAttempts) {
|
||||||
|
const reason = lastStatus !== undefined
|
||||||
|
? `HTTP ${lastStatus}`
|
||||||
|
: (lastError instanceof Error ? lastError.message : String(lastError))
|
||||||
|
log(`Retrying asset ${asset.id} (attempt ${attempt + 1}/${maxAttempts}) after ${reason}`)
|
||||||
|
// Linear backoff between attempts to avoid hammering a struggling server.
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 500 * attempt))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!buffer) {
|
||||||
|
fail({ asset, url, status: lastStatus, error: lastError })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (aborted) return
|
||||||
|
archive.append(buffer, { name: this.getFilename(asset) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await Promise.all(Array.from({ length: concurrency }, () => worker()))
|
||||||
|
|
||||||
|
if (aborted && failure) {
|
||||||
|
const f = failure as { asset: Asset, url: string, status?: number, error?: unknown }
|
||||||
|
const detail = f.status !== undefined
|
||||||
|
? `HTTP ${f.status}`
|
||||||
|
: (f.error instanceof Error ? f.error.message : String(f.error))
|
||||||
|
log(`Aborting zip download for share ${share.key}: failed to fetch asset ${f.asset.id} from ${f.url} (${detail})`)
|
||||||
|
archive.abort()
|
||||||
|
res.destroy()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
await archive.finalize()
|
await archive.finalize()
|
||||||
archive.on('end', () => res.end())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user