Clean up any broken staged downloads
This commit is contained in:
+4
-1
@@ -13,7 +13,7 @@ import {
|
|||||||
} from './immich'
|
} from './immich'
|
||||||
import crypto from 'crypto'
|
import crypto from 'crypto'
|
||||||
import { assetBuffer } from './stream/asset'
|
import { assetBuffer } from './stream/asset'
|
||||||
import { downloadAssets } from './stream/download'
|
import { downloadAssets, sweepStaleStagingDirs } from './stream/download'
|
||||||
import dayjs from 'dayjs'
|
import dayjs from 'dayjs'
|
||||||
import { NextFunction, Request, Response } from 'express-serve-static-core'
|
import { NextFunction, Request, Response } from 'express-serve-static-core'
|
||||||
import { Asset, AssetType, ImageSize, KeyType } from './types'
|
import { Asset, AssetType, ImageSize, KeyType } from './types'
|
||||||
@@ -282,4 +282,7 @@ process.on('SIGTERM', () => {
|
|||||||
const port = Number(process.env.IPP_PORT) || 3000
|
const port = Number(process.env.IPP_PORT) || 3000
|
||||||
const server = app.listen(port, () => {
|
const server = app.listen(port, () => {
|
||||||
console.log(dayjs().format() + ' Server started on port ' + port)
|
console.log(dayjs().format() + ' Server started on port ' + port)
|
||||||
|
// Clean up any zip-download staging dirs left behind by a previous
|
||||||
|
// run that crashed before its finally block could run
|
||||||
|
sweepStaleStagingDirs().catch(e => console.error('sweepStaleStagingDirs failed:', e))
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ import { getFilename } from '../gallery/filename'
|
|||||||
import { createLimiter } from '../utils/limiter'
|
import { createLimiter } from '../utils/limiter'
|
||||||
import { createIdleTimeoutStream } from '../utils/idleTimeoutStream'
|
import { createIdleTimeoutStream } from '../utils/idleTimeoutStream'
|
||||||
|
|
||||||
|
const STAGING_DIR_PREFIX = 'ipp-zip-'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Download all assets in a share as a zip file.
|
* Download all assets in a share as a zip file.
|
||||||
*/
|
*/
|
||||||
@@ -23,6 +25,29 @@ export async function downloadAll (res: Response, share: SharedLink) {
|
|||||||
await downloadAssets(res, share, share.assets)
|
await downloadAssets(res, share, share.assets)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete staging directories left over from a prior run that crashed before
|
||||||
|
* its `finally` block could clean up. Intended to run once at startup.
|
||||||
|
*
|
||||||
|
* A live download holds its staging dir open for the duration of the zip;
|
||||||
|
* `maxAgeMs` should comfortably exceed any realistic download time so we
|
||||||
|
* never delete a dir from a still-running download in another worker.
|
||||||
|
*/
|
||||||
|
export async function sweepStaleStagingDirs (maxAgeMs = 60 * 60 * 1000) {
|
||||||
|
const root = tmpdir()
|
||||||
|
const cutoff = Date.now() - maxAgeMs
|
||||||
|
const entries = await fs.readdir(root).catch(() => [] as string[])
|
||||||
|
for (const name of entries) {
|
||||||
|
if (!name.startsWith(STAGING_DIR_PREFIX)) continue
|
||||||
|
const path = join(root, name)
|
||||||
|
const stat = await fs.stat(path).catch(() => null)
|
||||||
|
if (!stat || stat.mtimeMs >= cutoff) continue
|
||||||
|
await fs.rm(path, { recursive: true, force: true }).catch(e => {
|
||||||
|
log.warn(`Failed to sweep stale staging dir ${path}: ${e instanceof Error ? e.message : String(e)}`)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type StagedAsset = { tempfile: string, asset: Asset, endpoint: ImageEndpoint }
|
type StagedAsset = { tempfile: string, asset: Asset, endpoint: ImageEndpoint }
|
||||||
type Failure = { asset: Asset, url: string, status?: number, error?: unknown }
|
type Failure = { asset: Asset, url: string, status?: number, error?: unknown }
|
||||||
type StageOutcome = StagedAsset | { failure: Failure } | null
|
type StageOutcome = StagedAsset | { failure: Failure } | null
|
||||||
@@ -70,7 +95,7 @@ export async function downloadAssets (res: Response, share: SharedLink, assets:
|
|||||||
archive.pipe(res)
|
archive.pipe(res)
|
||||||
|
|
||||||
const options: StagingOptions = {
|
const options: StagingOptions = {
|
||||||
stagingDir: await fs.mkdtemp(join(tmpdir(), 'ipp-zip-')),
|
stagingDir: await fs.mkdtemp(join(tmpdir(), STAGING_DIR_PREFIX)),
|
||||||
concurrency: Math.max(1, getConfigOption('ipp.downloadFromImmichConcurrencyLimit', 8) as number),
|
concurrency: Math.max(1, getConfigOption('ipp.downloadFromImmichConcurrencyLimit', 8) as number),
|
||||||
maxAttempts: 3,
|
maxAttempts: 3,
|
||||||
headerTimeoutMs: 20_000,
|
headerTimeoutMs: 20_000,
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { promises as fs } from 'fs'
|
||||||
|
import { tmpdir } from 'os'
|
||||||
|
import { join } from 'path'
|
||||||
|
import { sweepStaleStagingDirs } from '../src/stream/download'
|
||||||
|
|
||||||
|
const PREFIX = 'ipp-zip-'
|
||||||
|
|
||||||
|
async function makeDir (name: string, mtimeMsAgo: number) {
|
||||||
|
const path = join(tmpdir(), name)
|
||||||
|
await fs.mkdir(path, { recursive: true })
|
||||||
|
await fs.writeFile(join(path, 'inside.bin'), 'x')
|
||||||
|
const mtime = new Date(Date.now() - mtimeMsAgo)
|
||||||
|
await fs.utimes(path, mtime, mtime)
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
async function exists (path: string) {
|
||||||
|
try {
|
||||||
|
await fs.access(path)
|
||||||
|
return true
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const created: string[] = []
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await Promise.all(created.splice(0).map(p => fs.rm(p, { recursive: true, force: true }).catch(() => { /* best effort */ })))
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('sweepStaleStagingDirs', () => {
|
||||||
|
it('deletes stale staging dirs older than the cutoff', async () => {
|
||||||
|
const stale = await makeDir(PREFIX + 'stale-' + process.pid + '-' + Math.random().toString(36).slice(2), 2 * 60 * 60 * 1000)
|
||||||
|
created.push(stale)
|
||||||
|
await sweepStaleStagingDirs(60 * 60 * 1000)
|
||||||
|
expect(await exists(stale)).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('preserves staging dirs younger than the cutoff', async () => {
|
||||||
|
const fresh = await makeDir(PREFIX + 'fresh-' + process.pid + '-' + Math.random().toString(36).slice(2), 10 * 60 * 1000)
|
||||||
|
created.push(fresh)
|
||||||
|
await sweepStaleStagingDirs(60 * 60 * 1000)
|
||||||
|
expect(await exists(fresh)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('ignores directories that do not match the staging prefix', async () => {
|
||||||
|
const unrelated = await makeDir('not-ipp-zip-' + process.pid + '-' + Math.random().toString(36).slice(2), 24 * 60 * 60 * 1000)
|
||||||
|
created.push(unrelated)
|
||||||
|
await sweepStaleStagingDirs(60 * 60 * 1000)
|
||||||
|
expect(await exists(unrelated)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('uses its default 1h cutoff when called with no argument', async () => {
|
||||||
|
const stale = await makeDir(PREFIX + 'default-' + process.pid + '-' + Math.random().toString(36).slice(2), 2 * 60 * 60 * 1000)
|
||||||
|
const fresh = await makeDir(PREFIX + 'default2-' + process.pid + '-' + Math.random().toString(36).slice(2), 30 * 60 * 1000)
|
||||||
|
created.push(stale, fresh)
|
||||||
|
await sweepStaleStagingDirs()
|
||||||
|
expect(await exists(stale)).toBe(false)
|
||||||
|
expect(await exists(fresh)).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user