Initial work on upload app server
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
import { getConfigOption, getNumericConfigOption } from '@ipp/core'
|
||||
|
||||
const MB = 1024 * 1024
|
||||
|
||||
/**
|
||||
* Path the share routes are mounted under, from `PUBLIC_BASE_URL`: `''` for
|
||||
* an own-hostname deployment, `/upload` for `https://photos.example.com/upload`.
|
||||
*/
|
||||
export function basePathFrom (publicBaseUrl: string | undefined): string {
|
||||
if (!publicBaseUrl) return ''
|
||||
try {
|
||||
return new URL(publicBaseUrl).pathname.replace(/\/+$/, '')
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
/** IPP's public URL from `GALLERY_URL`, without a trailing slash, or undefined when unset. */
|
||||
export function galleryUrl (): string | undefined {
|
||||
return (process.env.GALLERY_URL || '').replace(/\/+$/, '') || undefined
|
||||
}
|
||||
|
||||
/** Largest single file accepted, in bytes, from `ipp.upload.maxFileSize` (MB). */
|
||||
export function maxFileSize (): number {
|
||||
return Math.max(1, getNumericConfigOption('ipp.upload.maxFileSize', 500)) * MB
|
||||
}
|
||||
|
||||
/** Files per minute per visitor address and share, from `ipp.upload.rateLimit`. `0` is no limit. */
|
||||
export function rateLimit (): number {
|
||||
return getNumericConfigOption('ipp.upload.rateLimit', 60)
|
||||
}
|
||||
|
||||
/** Bytes accepted per share per hour, from `ipp.upload.byteBudget` (MB). `0` is no limit. */
|
||||
export function byteBudget (): number {
|
||||
return Math.max(0, getNumericConfigOption('ipp.upload.byteBudget', 10240)) * MB
|
||||
}
|
||||
|
||||
/** Prefix for every stored filename, from `ipp.upload.filenamePrefix`. */
|
||||
export function filenamePrefix (): string {
|
||||
const configured = getConfigOption('ipp.upload.filenamePrefix', 'ipp_upload_')
|
||||
return typeof configured === 'string' ? configured : ''
|
||||
}
|
||||
|
||||
/** Notification webhook settings from `ipp.upload.notify*`, or undefined when no URL is set. */
|
||||
export function notifyConfig (): { url: string, headers: Record<string, string>, timeoutMs: number } | undefined {
|
||||
const url = getConfigOption('ipp.upload.notifyUrl', '')
|
||||
if (typeof url !== 'string' || !url) return undefined
|
||||
const headers: Record<string, string> = {}
|
||||
const configured = getConfigOption('ipp.upload.notifyHeaders', {})
|
||||
if (configured && typeof configured === 'object' && !Array.isArray(configured)) {
|
||||
for (const [name, value] of Object.entries(configured as Record<string, unknown>)) {
|
||||
if (typeof value === 'string') headers[name] = value
|
||||
}
|
||||
}
|
||||
return { url, headers, timeoutMs: getNumericConfigOption('ipp.upload.notifyTimeout', 10_000) }
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { sanitize } from '@ipp/core'
|
||||
|
||||
const MAX_FILENAME_LENGTH = 254
|
||||
|
||||
/**
|
||||
* Turn the visitor-supplied filename into the name Immich will store, with
|
||||
* `prefix` applied once. Returns undefined when nothing usable is left, or
|
||||
* when there is no extension: Immich types an asset by its extension and
|
||||
* would refuse it.
|
||||
*/
|
||||
export function uploadFilename (raw: unknown, prefix: string): string | undefined {
|
||||
if (typeof raw !== 'string') return undefined
|
||||
const base = raw.split(/[\\/]/).pop() || ''
|
||||
// Split the extension off first: `sanitize` truncates at 254 characters,
|
||||
// which on a very long name would cut the extension away.
|
||||
const parts = base.match(/^(.*)(\.[A-Za-z0-9]{1,16})$/)
|
||||
if (!parts) return undefined
|
||||
const extension = parts[2]
|
||||
const stem = sanitize(parts[1]).trim()
|
||||
// An empty stem is a dotfile; a leading dot is a traversal remnant.
|
||||
if (!stem || stem.startsWith('.')) return undefined
|
||||
|
||||
const cleanPrefix = sanitize(prefix).trim()
|
||||
// Don't stack the prefix on a file that already carries it
|
||||
const wanted = stem.startsWith(cleanPrefix) ? '' : cleanPrefix
|
||||
const room = MAX_FILENAME_LENGTH - wanted.length - extension.length
|
||||
return wanted + stem.slice(0, Math.max(1, room)) + extension
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Readable } from 'stream'
|
||||
import crypto from 'crypto'
|
||||
import { apiUrl, authHeaders, buildUrl, log, streamingRequestInit } from '@ipp/core'
|
||||
import { UploadPermit, assertPermit } from './gate'
|
||||
|
||||
/**
|
||||
* Everything forwarded about one file, already validated by `receive.ts`.
|
||||
* `filename` is sanitised and carries an extension; `mimeType` is the bare
|
||||
* type with no parameters.
|
||||
*/
|
||||
export interface IncomingUpload {
|
||||
filename: string
|
||||
mimeType: string
|
||||
fileCreatedAt: string
|
||||
fileModifiedAt: string
|
||||
/** Raw file bytes, streamed straight through and never buffered. */
|
||||
body: Readable
|
||||
/** The declared `Content-Length`; the upload aborts past it. */
|
||||
maxBytes: number
|
||||
}
|
||||
|
||||
export type ForwardResult =
|
||||
| { ok: true, status: 'created' | 'duplicate', id: string, bytes: number }
|
||||
| { ok: false, status: number, reason: string }
|
||||
|
||||
export const NOT_ACCEPTING_UPLOADS = 'This share does not accept uploads'
|
||||
|
||||
/** Thrown by the body generator when an upload runs past `maxBytes`. */
|
||||
class UploadTooLargeError extends Error {}
|
||||
|
||||
/**
|
||||
* Serialise one multipart/form-data body for Immich's `POST /assets`. The
|
||||
* field set is fixed, so it is written by hand. The async generator pulls a
|
||||
* chunk only when the socket to Immich is ready for it.
|
||||
*
|
||||
* Only the two timestamps and the file are sent, so fields Immich would
|
||||
* otherwise honour (`visibility`, `livePhotoVideoId`, `metadata`,
|
||||
* `sidecarData`) cannot be set by a visitor. The filename rides on the
|
||||
* part's own Content-Disposition, which is where Immich reads it from.
|
||||
*/
|
||||
function multipartBody (boundary: string, upload: IncomingUpload, counter: { bytes: number }): Readable {
|
||||
const fields: Record<string, string> = {
|
||||
fileCreatedAt: upload.fileCreatedAt,
|
||||
fileModifiedAt: upload.fileModifiedAt
|
||||
}
|
||||
const part = (headers: string) => Buffer.from(`--${boundary}\r\n${headers}\r\n\r\n`)
|
||||
|
||||
async function * generate () {
|
||||
for (const [name, value] of Object.entries(fields)) {
|
||||
yield part(`Content-Disposition: form-data; name="${name}"`)
|
||||
yield Buffer.from(value + '\r\n')
|
||||
}
|
||||
yield part(
|
||||
`Content-Disposition: form-data; name="assetData"; filename="${upload.filename}"\r\n` +
|
||||
`Content-Type: ${upload.mimeType}`
|
||||
)
|
||||
for await (const chunk of upload.body) {
|
||||
counter.bytes += chunk.length
|
||||
if (counter.bytes > upload.maxBytes) throw new UploadTooLargeError()
|
||||
yield chunk
|
||||
}
|
||||
yield Buffer.from(`\r\n--${boundary}--\r\n`)
|
||||
}
|
||||
|
||||
return Readable.from(generate())
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Immich's answer to an upload onto the reply the visitor gets. 201 is a
|
||||
* new asset; 200 is a checksum match with an asset the owner already has,
|
||||
* which Immich still adds to the album or link. 400 is Immich refusing the
|
||||
* file itself. 401 and 403 mean the share no longer accepts uploads.
|
||||
*/
|
||||
export function replyForImmichStatus (status: number, body?: { id?: string, status?: string } | null): ForwardResult {
|
||||
if (status === 200 || status === 201) {
|
||||
return { ok: true, status: body?.status === 'duplicate' ? 'duplicate' : 'created', id: body?.id || '', bytes: 0 }
|
||||
}
|
||||
if (status === 400) return { ok: false, status: 400, reason: 'Immich would not accept this file' }
|
||||
if (status === 401 || status === 403) return { ok: false, status: 403, reason: NOT_ACCEPTING_UPLOADS }
|
||||
return { ok: false, status: 502, reason: 'Immich rejected the upload' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Send one upload to Immich as the share, and report what Immich made of it.
|
||||
* Authentication is the visitor's share key (plus the password token for a
|
||||
* password share); Immich files the asset into the album or the link itself.
|
||||
* Errors are returned, except a permit failure, which is a bug.
|
||||
*/
|
||||
export async function forwardUpload (permit: UploadPermit, upload: IncomingUpload, signal: AbortSignal): Promise<ForwardResult> {
|
||||
assertPermit(permit)
|
||||
|
||||
const boundary = '----IPPUpload' + crypto.randomBytes(16).toString('hex')
|
||||
const counter = { bytes: 0 }
|
||||
const url = buildUrl(apiUrl() + '/assets', { [permit.keyType]: permit.key })
|
||||
|
||||
let res: globalThis.Response
|
||||
try {
|
||||
res = await fetch(url, streamingRequestInit({
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'multipart/form-data; boundary=' + boundary,
|
||||
...(await authHeaders(permit.keyType, permit.key, permit.password))
|
||||
},
|
||||
signal
|
||||
}, multipartBody(boundary, upload, counter)))
|
||||
} catch (e) {
|
||||
// The generator's throw surfaces here as the fetch rejecting
|
||||
if (causedBy(e, UploadTooLargeError)) return { ok: false, status: 413, reason: 'File is larger than declared' }
|
||||
if (signal.aborted) return { ok: false, status: 499, reason: 'Visitor cancelled the upload' }
|
||||
log.warn('Upload to Immich failed for ' + upload.filename + ': ' + errorText(e))
|
||||
return { ok: false, status: 502, reason: 'Could not reach Immich' }
|
||||
}
|
||||
|
||||
const body = await res.json().catch(() => null) as { id?: string, status?: string, message?: string } | null
|
||||
const result = replyForImmichStatus(res.status, body)
|
||||
if (result.ok) return { ...result, bytes: counter.bytes }
|
||||
log.warn('Immich refused upload ' + upload.filename + ' with status ' + res.status +
|
||||
(body?.message ? ': ' + body.message : ''))
|
||||
return result
|
||||
}
|
||||
|
||||
/** Walk the `cause` chain: fetch wraps the body generator's error. */
|
||||
function causedBy (e: unknown, type: new () => Error): boolean {
|
||||
let current: unknown = e
|
||||
while (current instanceof Error) {
|
||||
if (current instanceof type) return true
|
||||
current = (current as { cause?: unknown }).cause
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function errorText (e: unknown): string {
|
||||
return e instanceof Error ? e.message : String(e)
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
The upload authorisation gate. The upload is the only place this codebase
|
||||
writes to Immich, so the check is not left to a caller remembering it:
|
||||
|
||||
1. `authoriseUpload()` - per request, once the share is resolved. Returns
|
||||
an `UploadPermit`, which `forwardUpload` cannot be called without.
|
||||
2. `assertPermit()` - immediately before the request to Immich is built.
|
||||
|
||||
The gate is the owner's "Allow public user to upload" toggle on the share,
|
||||
read from a share lookup at most 60 s old. Immich enforces the same toggle
|
||||
on every upload, so a mistake here still cannot upload to a share whose
|
||||
owner said no.
|
||||
*/
|
||||
|
||||
import { KeyType, SharedLink, slugLinksDisabled } from '@ipp/core'
|
||||
|
||||
/*
|
||||
A real symbol rather than a type-only brand, so `assertPermit` has
|
||||
something to test once types are erased. Not exported and not in the
|
||||
global registry, so a permit cannot be built outside this file.
|
||||
*/
|
||||
const permitBrand: unique symbol = Symbol('ipp.uploadPermit')
|
||||
|
||||
/**
|
||||
* Proof that a share passed the upload gate, plus the share-scoped
|
||||
* credentials the forwarder needs. Obtainable only from `authoriseUpload`.
|
||||
*/
|
||||
export interface UploadPermit {
|
||||
readonly [permitBrand]: true
|
||||
readonly link: SharedLink
|
||||
/** The key exactly as the visitor's URL carried it (a key or a slug). */
|
||||
readonly key: string
|
||||
readonly keyType: KeyType
|
||||
/** Share password from the session cookie, for password-protected shares. */
|
||||
readonly password?: string
|
||||
}
|
||||
|
||||
/** Whether this share accepts uploads through this key type right now. */
|
||||
export function uploadAllowed (link: SharedLink, keyType: KeyType): boolean {
|
||||
return link.allowUpload === true && !slugLinksDisabled(keyType)
|
||||
}
|
||||
|
||||
/** Issue a permit, or null when this share does not accept uploads. */
|
||||
export function authoriseUpload (link: SharedLink, key: string, keyType: KeyType, password?: string): UploadPermit | null {
|
||||
if (!uploadAllowed(link, keyType)) return null
|
||||
return { [permitBrand]: true, link, key, keyType, password } as UploadPermit
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-check the gate at the point of use. Throws, because reaching here
|
||||
* without a valid permit is a bug, not a visitor error.
|
||||
*/
|
||||
export function assertPermit (permit: UploadPermit): void {
|
||||
if (!permit || permit[permitBrand] !== true) {
|
||||
throw new Error('Upload attempted without a permit')
|
||||
}
|
||||
if (!uploadAllowed(permit.link, permit.keyType)) {
|
||||
throw new Error('Upload attempted for a share that does not allow uploads')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { Transform } from 'stream'
|
||||
|
||||
/** The error an idle stream is destroyed with. */
|
||||
export class IdleTimeoutError extends Error {}
|
||||
|
||||
/**
|
||||
* A pass-through Transform that destroys itself if no data flows through for
|
||||
* `idleMs`. The timer is set when the transform is created and reset on every
|
||||
* chunk, so a slow-but-steady download (large video over a slow link) keeps
|
||||
* going, while a genuinely stalled connection still fails fast.
|
||||
*/
|
||||
export function createIdleTimeoutStream (idleMs: number): Transform {
|
||||
let timer: NodeJS.Timeout | undefined
|
||||
const transform: Transform = new Transform({
|
||||
transform (chunk, _, cb) {
|
||||
if (timer) clearTimeout(timer)
|
||||
timer = setTimeout(() => transform.destroy(new IdleTimeoutError(`No data received for ${idleMs}ms`)), idleMs)
|
||||
cb(null, chunk)
|
||||
},
|
||||
flush (cb) {
|
||||
if (timer) clearTimeout(timer)
|
||||
cb()
|
||||
}
|
||||
})
|
||||
// Arm the timer immediately so a response that returns headers but never
|
||||
// sends a body also times out.
|
||||
timer = setTimeout(() => transform.destroy(new IdleTimeoutError(`No data received for ${idleMs}ms`)), idleMs)
|
||||
return transform
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import 'dotenv/config'
|
||||
import express, { Router } from 'express'
|
||||
import { Request } from 'express-serve-static-core'
|
||||
import { resolve } from 'path'
|
||||
import dayjs from 'dayjs'
|
||||
import { h } from 'preact'
|
||||
import {
|
||||
addNoStoreHeaders,
|
||||
asyncHandler,
|
||||
CORE_PUBLIC_DIR,
|
||||
decodeCookie,
|
||||
enforceMinimumImmichVersion,
|
||||
errorHandler,
|
||||
getKeyTypeFromShare,
|
||||
healthcheck,
|
||||
invalidPasswordResponse,
|
||||
isKey,
|
||||
KeyType,
|
||||
loadConfig,
|
||||
Password,
|
||||
renderPage,
|
||||
respondToInvalidRequest,
|
||||
sessionMiddleware,
|
||||
SharedLink,
|
||||
slugLinksDisabled,
|
||||
title,
|
||||
unlockHandler
|
||||
} from '@ipp/core'
|
||||
import { basePathFrom, galleryUrl, maxFileSize } from './config'
|
||||
import { uploadAllowed } from './gate'
|
||||
import { createUploadReceiver } from './receive'
|
||||
import { getShare } from './share'
|
||||
import { ASSET_VERSION } from './version'
|
||||
import { UploadPage } from './view/upload'
|
||||
|
||||
// Must run before any code that calls getConfigOption. The bundled
|
||||
// config.json sits one level above this file, which is /app/config.json in
|
||||
// the image.
|
||||
loadConfig({ defaultPath: resolve(__dirname, '../config.json') })
|
||||
|
||||
const basePath = basePathFrom(process.env.PUBLIC_BASE_URL)
|
||||
const receiveUpload = createUploadReceiver()
|
||||
const inProduction = process.env.NODE_ENV === 'production'
|
||||
|
||||
const app = express()
|
||||
app.disable('x-powered-by')
|
||||
|
||||
// Reachable at the root whatever the mount prefix, so one compose healthcheck fits both routing shapes
|
||||
app.get(['/healthcheck', '/share/healthcheck'], asyncHandler(healthcheck))
|
||||
|
||||
const router = Router()
|
||||
|
||||
for (const dir of ['public', CORE_PUBLIC_DIR]) {
|
||||
router.use('/share/static/' + ASSET_VERSION, express.static(dir, { immutable: inProduction, maxAge: inProduction ? '365d' : 0 }))
|
||||
router.use('/share/static', express.static(dir))
|
||||
}
|
||||
|
||||
// Everything past the static assets is never cached
|
||||
router.use((_req, res, next) => {
|
||||
addNoStoreHeaders(res)
|
||||
next()
|
||||
})
|
||||
router.use(sessionMiddleware({ name: 'upload-session', path: basePath || '/' }))
|
||||
|
||||
type ShareResolution =
|
||||
| { ok: true, link: SharedLink, keyType: KeyType }
|
||||
| { ok: false, reason: string, passwordRequired?: boolean }
|
||||
|
||||
/** Validate the share in the URL against Immich, with the password from the session cookie. */
|
||||
async function resolveShare (req: Request): Promise<ShareResolution> {
|
||||
const keyType = getKeyTypeFromShare(req.params.shareType)
|
||||
if (slugLinksDisabled(keyType)) return { ok: false, reason: 'Slug links are disabled in config.json' }
|
||||
if (!isKey(req.params.key)) return { ok: false, reason: 'Invalid key for ' + req.path }
|
||||
const share = await getShare(req.params.key, keyType, req.password)
|
||||
if (!share.valid) return { ok: false, reason: 'Invalid share link' }
|
||||
if (share.passwordRequired || !share.link) return { ok: false, reason: 'Password required', passwordRequired: true }
|
||||
return { ok: true, link: share.link, keyType }
|
||||
}
|
||||
|
||||
/*
|
||||
* [ROUTE] Upload page, or the password page for a locked share
|
||||
*/
|
||||
router.get('/:shareType(share|s)/:key', decodeCookie, asyncHandler(async (req, res) => {
|
||||
const key = req.params.key
|
||||
const resolved = await resolveShare(req)
|
||||
if (!resolved.ok && resolved.passwordRequired) {
|
||||
if (req.password) invalidPasswordResponse(req, res, key)
|
||||
res.send(renderPage(h(Password, { shareKey: key, notifyInvalidPassword: !!req.password, basePath })))
|
||||
return
|
||||
}
|
||||
if (!resolved.ok || !uploadAllowed(resolved.link, resolved.keyType)) {
|
||||
respondToInvalidRequest(res, 404, resolved.ok ? 'Share does not accept uploads' : resolved.reason)
|
||||
return
|
||||
}
|
||||
const gallery = galleryUrl()
|
||||
res.send(renderPage(h(UploadPage, {
|
||||
config: {
|
||||
uploadUrl: `${basePath}/${req.params.shareType}/${encodeURIComponent(key)}/upload`,
|
||||
maxFileSize: maxFileSize(),
|
||||
title: title(resolved.link)
|
||||
},
|
||||
basePath,
|
||||
galleryLink: gallery && `${gallery}/${req.params.shareType}/${encodeURIComponent(key)}`
|
||||
})))
|
||||
}))
|
||||
|
||||
/*
|
||||
* [ROUTE] Password unlock from the password page
|
||||
*/
|
||||
router.post('/share/unlock', express.json({ limit: '10kb' }), unlockHandler)
|
||||
|
||||
/*
|
||||
* [ROUTE] One file from a visitor, streamed to Immich
|
||||
*/
|
||||
router.post('/:shareType(share|s)/:key/upload', decodeCookie, asyncHandler(async (req, res) => {
|
||||
const resolved = await resolveShare(req)
|
||||
if (!resolved.ok) {
|
||||
res.set('Connection', 'close')
|
||||
respondToInvalidRequest(res, 404, resolved.reason)
|
||||
return
|
||||
}
|
||||
await receiveUpload(req, res, resolved.link, resolved.keyType)
|
||||
}))
|
||||
|
||||
router.get(['/healthcheck', '/share/healthcheck'], asyncHandler(healthcheck))
|
||||
|
||||
app.use(basePath || '/', router)
|
||||
|
||||
/*
|
||||
* Everything else, including `/`, gets the 404 policy
|
||||
*/
|
||||
app.all('*', (req, res) => {
|
||||
addNoStoreHeaders(res)
|
||||
respondToInvalidRequest(res, 404, 'Invalid route ' + req.path)
|
||||
})
|
||||
|
||||
app.use(errorHandler)
|
||||
|
||||
process.on('uncaughtException', (err) => {
|
||||
console.error('There was an uncaught error', err)
|
||||
server.close()
|
||||
process.exit(1)
|
||||
})
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason)
|
||||
})
|
||||
process.on('SIGTERM', () => {
|
||||
console.log('Received SIGTERM. Gracefully shutting down...')
|
||||
server.close()
|
||||
process.exit(0)
|
||||
})
|
||||
|
||||
const port = Number(process.env.IPP_PORT) || 3000
|
||||
const server = app.listen(port, () => {
|
||||
console.log(dayjs().format() + ' Upload service started on port ' + port + (basePath ? ' under ' + basePath : ''))
|
||||
enforceMinimumImmichVersion().catch(() => {})
|
||||
})
|
||||
// A large video on a phone uplink outlasts any whole-request limit; the body has an idle timeout instead
|
||||
server.requestTimeout = 0
|
||||
@@ -0,0 +1,54 @@
|
||||
import { TtlLruCache } from '@ipp/core'
|
||||
|
||||
export type Admission =
|
||||
| { ok: true, release: () => void }
|
||||
| { ok: false, retryAfterSeconds: number }
|
||||
|
||||
interface Window {
|
||||
used: number
|
||||
resetAt: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixed-window budget: each key may admit up to `limit` units per window.
|
||||
* The rate limit admits one unit per file; the byte budget admits a file's
|
||||
* declared length, and releases it if the upload then fails.
|
||||
*
|
||||
* Backed by TtlLruCache, so a flood of distinct keys evicts instead of
|
||||
* growing without limit. The window lives on the entry rather than on the
|
||||
* cache TTL, so admitting inside a window never extends it. A `limit` of 0
|
||||
* or less means no limit.
|
||||
*/
|
||||
export function createWindowBudget (opts: { limit: number, windowMs: number, maxKeys?: number }): (key: string, amount?: number) => Admission {
|
||||
const windows = new TtlLruCache<Window>({
|
||||
// Twice the window, so the cache never drops an entry while its window is open
|
||||
ttlMs: opts.windowMs * 2,
|
||||
max: opts.maxKeys ?? 5000
|
||||
})
|
||||
const unlimited: Admission = { ok: true, release: () => {} }
|
||||
|
||||
return function admit (key: string, amount = 1): Admission {
|
||||
if (opts.limit <= 0) return unlimited
|
||||
const now = Date.now()
|
||||
let window = windows.get(key)
|
||||
if (!window || window.resetAt <= now) {
|
||||
window = { used: 0, resetAt: now + opts.windowMs }
|
||||
windows.set(key, window)
|
||||
}
|
||||
if (window.used + amount > opts.limit) {
|
||||
return { ok: false, retryAfterSeconds: Math.max(1, Math.ceil((window.resetAt - now) / 1000)) }
|
||||
}
|
||||
window.used += amount
|
||||
const admitted = window
|
||||
let released = false
|
||||
return {
|
||||
ok: true,
|
||||
release: () => {
|
||||
// A release after its window closed has nothing left to give back
|
||||
if (released || admitted.resetAt <= Date.now()) return
|
||||
released = true
|
||||
admitted.used -= amount
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { AlbumType, log, SharedLink, title } from '@ipp/core'
|
||||
import { formatSize } from './shared/upload'
|
||||
import { notifyConfig } from './config'
|
||||
|
||||
/*
|
||||
Optional upload notification: one JSON POST per stored file to
|
||||
`ipp.upload.notifyUrl`, for ntfy, Gotify, Home Assistant and the like.
|
||||
The payload never carries the share key, slug or password.
|
||||
*/
|
||||
|
||||
export interface UploadNotification {
|
||||
title: string
|
||||
message: string
|
||||
data: {
|
||||
event: 'upload'
|
||||
time: string
|
||||
share: { type: 'album' | 'individual', id: string, title: string }
|
||||
file: { name: string, size: number, type: string }
|
||||
asset: { id: string, status: 'created' | 'duplicate' }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the notification body for one stored file. `title` and `message` are
|
||||
* top level so a receiver expecting a flat shape shows something readable
|
||||
* with no mapping; everything structured lives under `data`.
|
||||
*/
|
||||
export function buildNotification (
|
||||
share: SharedLink,
|
||||
file: { name: string, size: number, type: string },
|
||||
asset: { id: string, status: 'created' | 'duplicate' }
|
||||
): UploadNotification {
|
||||
const isAlbum = share.type === AlbumType.album
|
||||
const shareTitle = title(share)
|
||||
return {
|
||||
title: 'Photo sent to ' + shareTitle,
|
||||
message: `${file.name} (${formatSize(file.size)}) was uploaded to ${shareTitle}.`,
|
||||
data: {
|
||||
event: 'upload',
|
||||
time: new Date().toISOString(),
|
||||
share: {
|
||||
type: isAlbum ? 'album' : 'individual',
|
||||
// An album share is named by its album, an individual share by the link
|
||||
id: (isAlbum ? share.album?.id : share.id) || '',
|
||||
title: shareTitle
|
||||
},
|
||||
file,
|
||||
asset
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Post the notification, if one is configured. Fire and forget: a timeout,
|
||||
* no retry, failures logged and swallowed. Only the URL's host is logged,
|
||||
* because ntfy topics and Gotify tokens live in the URL.
|
||||
*/
|
||||
export async function sendNotification (notification: UploadNotification): Promise<void> {
|
||||
const config = notifyConfig()
|
||||
if (!config) return
|
||||
try {
|
||||
const res = await fetch(config.url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', ...config.headers },
|
||||
body: JSON.stringify(notification),
|
||||
signal: AbortSignal.timeout(config.timeoutMs)
|
||||
})
|
||||
if (!res.ok) log.warn('Upload notification to ' + hostOf(config.url) + ' answered ' + res.status)
|
||||
} catch (e) {
|
||||
log.warn('Upload notification to ' + hostOf(config.url) + ' failed: ' + (e instanceof Error ? e.message : String(e)))
|
||||
}
|
||||
}
|
||||
|
||||
function hostOf (url: string): string {
|
||||
try {
|
||||
return new URL(url).host
|
||||
} catch {
|
||||
return 'an invalid URL'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import { pipeline } from 'stream'
|
||||
import { Request, Response } from 'express-serve-static-core'
|
||||
import dayjs from 'dayjs'
|
||||
import { abortOnClose, KeyType, log, respondToInvalidRequest, SharedLink } from '@ipp/core'
|
||||
import { byteBudget, filenamePrefix, maxFileSize, rateLimit } from './config'
|
||||
import { uploadFilename } from './filename'
|
||||
import { forwardUpload } from './forward'
|
||||
import { authoriseUpload } from './gate'
|
||||
import { createIdleTimeoutStream, IdleTimeoutError } from './idleTimeoutStream'
|
||||
import { createWindowBudget } from './limits'
|
||||
import { buildNotification, sendNotification } from './notify'
|
||||
import { formatSize, isUploadableType } from './shared/upload'
|
||||
|
||||
/*
|
||||
The visitor-facing half of the upload path: validate one request, then
|
||||
hand it to `forward.ts` with a permit from `gate.ts`.
|
||||
|
||||
Each file is its own POST whose body is the raw bytes, with the name and
|
||||
timestamps in the query string. The app re-encodes for Immich anyway, so
|
||||
it runs no multipart parser on visitor input, and nothing is buffered or
|
||||
written to disk.
|
||||
*/
|
||||
|
||||
// A body that sends nothing for this long is dropped
|
||||
const BODY_IDLE_MS = 60_000
|
||||
|
||||
/**
|
||||
* Answer an upload the server will not take, in the shape the page reads.
|
||||
* `Connection: close` stops a body still in flight costing its full bandwidth.
|
||||
*/
|
||||
function refuse (res: Response, status: number, reason: string): void {
|
||||
res.set('Connection', 'close')
|
||||
res.status(status).json({ error: reason })
|
||||
}
|
||||
|
||||
/** The declared `Content-Length`, or undefined when absent or malformed. */
|
||||
export function declaredLength (header: string | undefined): number | undefined {
|
||||
return header && /^\d+$/.test(header) ? Number(header) : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a visitor-supplied timestamp, falling back to now. Only a starting
|
||||
* point: Immich replaces it from EXIF when the file carries a better date.
|
||||
*/
|
||||
export function uploadTimestamp (raw: unknown): string {
|
||||
const parsed = typeof raw === 'string' && raw ? dayjs(raw) : undefined
|
||||
return (parsed?.isValid() ? parsed : dayjs()).toISOString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the handler for `POST <share>/upload`, with its rate limit and byte
|
||||
* budget read from config. Called once, after the config has loaded.
|
||||
*
|
||||
* Once the share has passed the gate, failures a visitor can act on answer
|
||||
* with a status and a short JSON reason rather than the 404 policy.
|
||||
*/
|
||||
export function createUploadReceiver () {
|
||||
// Keyed on address and share; behind a reverse proxy every visitor has the proxy's address
|
||||
const rate = createWindowBudget({ limit: rateLimit(), windowMs: 60_000 })
|
||||
const budget = createWindowBudget({ limit: byteBudget(), windowMs: 60 * 60_000 })
|
||||
const maxBytes = maxFileSize()
|
||||
const prefix = filenamePrefix()
|
||||
|
||||
return async function receiveUpload (req: Request, res: Response, link: SharedLink, keyType: KeyType): Promise<void> {
|
||||
const key = req.params.key
|
||||
const permit = authoriseUpload(link, key, keyType, req.password)
|
||||
if (!permit) {
|
||||
res.set('Connection', 'close')
|
||||
respondToInvalidRequest(res, 404, 'Share does not accept uploads')
|
||||
return
|
||||
}
|
||||
const shareId = link.id || key
|
||||
|
||||
const rated = rate(req.ip + ':' + shareId)
|
||||
if (!rated.ok) {
|
||||
res.set('Retry-After', String(rated.retryAfterSeconds))
|
||||
refuse(res, 429, 'Too many uploads. Wait a moment and try again.')
|
||||
return
|
||||
}
|
||||
|
||||
const length = declaredLength(req.headers['content-length'])
|
||||
if (length === undefined) {
|
||||
refuse(res, 411, 'The upload has no Content-Length')
|
||||
return
|
||||
}
|
||||
if (length > maxBytes) {
|
||||
refuse(res, 413, 'Larger than the ' + formatSize(maxBytes) + ' limit')
|
||||
return
|
||||
}
|
||||
|
||||
const filename = uploadFilename(req.query.filename, prefix)
|
||||
if (!filename) {
|
||||
refuse(res, 400, 'Missing or unusable filename')
|
||||
return
|
||||
}
|
||||
|
||||
const mimeType = (req.headers['content-type'] || '').split(';')[0].trim().toLowerCase()
|
||||
if (!isUploadableType(mimeType)) {
|
||||
refuse(res, 415, 'Only photos and videos can be sent')
|
||||
return
|
||||
}
|
||||
|
||||
// Reserved before streaming, so parallel uploads cannot overshoot the budget together
|
||||
const reserved = budget(shareId, length)
|
||||
if (!reserved.ok) {
|
||||
res.set('Retry-After', String(reserved.retryAfterSeconds))
|
||||
refuse(res, 413, 'This share has received its limit for the hour. Try again later.')
|
||||
return
|
||||
}
|
||||
|
||||
const body = pipeline(req, createIdleTimeoutStream(BODY_IDLE_MS), err => {
|
||||
if (err instanceof IdleTimeoutError) log('Dropped upload ' + filename + ': nothing received for ' + BODY_IDLE_MS / 1000 + ' s')
|
||||
})
|
||||
const result = await forwardUpload(permit, {
|
||||
filename,
|
||||
mimeType,
|
||||
fileCreatedAt: uploadTimestamp(req.query.fileCreatedAt),
|
||||
fileModifiedAt: uploadTimestamp(req.query.fileModifiedAt),
|
||||
body,
|
||||
maxBytes: length
|
||||
}, abortOnClose(res))
|
||||
|
||||
if (!result.ok) {
|
||||
reserved.release()
|
||||
// The visitor hung up; there is nobody left to answer
|
||||
if (result.status === 499) return
|
||||
refuse(res, result.status, result.reason)
|
||||
return
|
||||
}
|
||||
|
||||
log('Received upload ' + filename + ' for share ' + (link.slug || link.id) + ' (' + result.status + ')')
|
||||
res.json({ status: result.status })
|
||||
|
||||
// After the reply, so a slow webhook never holds up the visitor
|
||||
sendNotification(buildNotification(
|
||||
link,
|
||||
{ name: filename, size: result.bytes, type: mimeType },
|
||||
{ id: result.id, status: result.status }
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { cachedPromise, fetchSharedLink, KeyType, SharedLinkResult, TtlLruCache } from '@ipp/core'
|
||||
|
||||
/*
|
||||
Share lookups, cached for 60 s per (key type, key, password). The cache
|
||||
bounds how stale the `allowUpload` check can be when an owner turns the
|
||||
toggle off; Immich enforces the toggle on every upload regardless.
|
||||
*/
|
||||
const shareCache = new TtlLruCache<Promise<SharedLinkResult>>({ ttlMs: 60_000, max: 100 })
|
||||
|
||||
/** Look up a share through `@ipp/core`, sharing one Immich call across concurrent requests. */
|
||||
export function getShare (key: string, keyType: KeyType, password?: string): Promise<SharedLinkResult> {
|
||||
return cachedPromise(shareCache, `${keyType}:${key}:${password || ''}`, () => fetchSharedLink(key, keyType, password), result => result.valid)
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Returns a function that runs at most `limit` async tasks concurrently.
|
||||
* Tasks queue when the limit is reached and resume in FIFO order.
|
||||
*/
|
||||
export function createLimiter (limit: number) {
|
||||
let active = 0
|
||||
const queue: Array<() => void> = []
|
||||
return async function run<T> (fn: () => Promise<T>): Promise<T> {
|
||||
if (active >= limit) {
|
||||
await new Promise<void>(resolve => queue.push(resolve))
|
||||
}
|
||||
active++
|
||||
try {
|
||||
return await fn()
|
||||
} finally {
|
||||
active--
|
||||
const next = queue.shift()
|
||||
if (next) next()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
Upload rules the page client and the server both apply: the client filters
|
||||
with them before anything leaves the browser, the server decides with them.
|
||||
*/
|
||||
|
||||
/** `accept` attribute for the file picker: the camera roll on a phone. */
|
||||
export const UPLOAD_ACCEPT = 'image/*,video/*'
|
||||
|
||||
/**
|
||||
* Whether a file of this media type may be sent. Any image or video except
|
||||
* SVG, the one image type a browser may execute. Immich types an asset by its
|
||||
* extension and refuses anything outside its own list, so the finer check is
|
||||
* left to Immich.
|
||||
*/
|
||||
export function isUploadableType (mimeType: string): boolean {
|
||||
const type = (mimeType || '').split(';')[0].trim().toLowerCase()
|
||||
if (type === 'image/svg+xml') return false
|
||||
return type.startsWith('image/') || type.startsWith('video/')
|
||||
}
|
||||
|
||||
/*
|
||||
Types for files a desktop browser reports with an empty `File.type`: HEIC,
|
||||
HEIF, camera RAW and some video containers. Immich types the asset by its
|
||||
extension, so these only need to pass `isUploadableType`.
|
||||
*/
|
||||
const TYPE_BY_EXTENSION: Record<string, string> = {
|
||||
heic: 'image/heic',
|
||||
heif: 'image/heif',
|
||||
hif: 'image/heif',
|
||||
avif: 'image/avif',
|
||||
jxl: 'image/jxl',
|
||||
dng: 'image/x-adobe-dng',
|
||||
cr2: 'image/x-canon-cr2',
|
||||
cr3: 'image/x-canon-cr3',
|
||||
nef: 'image/x-nikon-nef',
|
||||
nrw: 'image/x-nikon-nrw',
|
||||
arw: 'image/x-sony-arw',
|
||||
raf: 'image/x-fuji-raf',
|
||||
orf: 'image/x-olympus-orf',
|
||||
rw2: 'image/x-panasonic-rw2',
|
||||
pef: 'image/x-pentax-pef',
|
||||
srw: 'image/x-samsung-srw',
|
||||
mts: 'video/mp2t',
|
||||
m2ts: 'video/mp2t',
|
||||
mkv: 'video/x-matroska',
|
||||
insv: 'video/mp4'
|
||||
}
|
||||
|
||||
/** The browser's type for a file, or one derived from its extension when the browser gives none. */
|
||||
export function fileType (name: string, browserType: string): string {
|
||||
if (browserType) return browserType
|
||||
const extension = name.split('.').pop()?.toLowerCase() || ''
|
||||
return TYPE_BY_EXTENSION[extension] || ''
|
||||
}
|
||||
|
||||
/** Bytes as a short human-readable string. */
|
||||
export function formatSize (bytes: number): string {
|
||||
if (bytes >= 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(1) + ' GB'
|
||||
if (bytes >= 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + ' MB'
|
||||
if (bytes >= 1024) return Math.round(bytes / 1024) + ' KB'
|
||||
return bytes + ' bytes'
|
||||
}
|
||||
|
||||
/** Page settings the server renders into the init JSON for the client. */
|
||||
export interface UploadPageConfig {
|
||||
/** Path of this share's upload endpoint, including any mount prefix. */
|
||||
uploadUrl: string
|
||||
/** Largest single file the server accepts, in bytes. */
|
||||
maxFileSize: number
|
||||
/** Share title, for the drop overlay and the completion message. */
|
||||
title: string
|
||||
}
|
||||
|
||||
/** Success body of `POST <share>/upload`. */
|
||||
export interface UploadResponse {
|
||||
status: 'created' | 'duplicate'
|
||||
}
|
||||
|
||||
/** Failure body of `POST <share>/upload`. */
|
||||
export interface UploadErrorResponse {
|
||||
error: string
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { join } from 'path'
|
||||
import { resolveAppVersion } from '@ipp/core'
|
||||
|
||||
/** URL-safe cache-busting segment for static asset paths. */
|
||||
export const ASSET_VERSION = encodeURIComponent(resolveAppVersion(join(__dirname, '..', 'package.json')))
|
||||
@@ -0,0 +1,47 @@
|
||||
import { jsonForInlineScript, ThemeScript } from '@ipp/core'
|
||||
import { UPLOAD_ACCEPT, UploadPageConfig } from '../shared/upload'
|
||||
import { ASSET_VERSION } from '../version'
|
||||
|
||||
export interface UploadPageProps {
|
||||
config: UploadPageConfig
|
||||
/** Path prefix the app is mounted under, without a trailing slash. Empty at the root. */
|
||||
basePath: string
|
||||
/** This share's gallery on IPP, when `GALLERY_URL` is set. */
|
||||
galleryLink?: string
|
||||
}
|
||||
|
||||
/** The upload page for one share: the share title, the picker, and the list the client fills. */
|
||||
export function UploadPage ({ config, basePath, galleryLink }: UploadPageProps) {
|
||||
const staticPath = `${basePath}/share/static/${ASSET_VERSION}`
|
||||
return (
|
||||
<html lang="en">
|
||||
<head>
|
||||
<ThemeScript/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<meta name="robots" content="noindex"/>
|
||||
<title>{'Send photos to ' + config.title}</title>
|
||||
<link rel="icon" href={basePath + '/share/static/favicon.ico'} type="image/x-icon"/>
|
||||
<link type="text/css" rel="stylesheet" href={staticPath + '/pico.min.css'}/>
|
||||
<link type="text/css" rel="stylesheet" href={staticPath + '/theme.css'}/>
|
||||
<link type="text/css" rel="stylesheet" href={staticPath + '/style.css'}/>
|
||||
</head>
|
||||
<body>
|
||||
<main class="container upload-page">
|
||||
<h1>Send photos to <em>{config.title}</em></h1>
|
||||
<button id="upload-choose" type="button" class="upload-choose">Choose photos</button>
|
||||
<input id="upload-input" type="file" multiple accept={UPLOAD_ACCEPT} hidden/>
|
||||
{galleryLink && <p><a class="upload-back" href={galleryLink}>Back to gallery</a></p>}
|
||||
<section id="upload-batch" hidden aria-live="polite">
|
||||
<h2 id="upload-status"></h2>
|
||||
<ul id="upload-list"></ul>
|
||||
</section>
|
||||
</main>
|
||||
<div id="upload-drop" hidden>
|
||||
<p>Drop to send photos to <em>{config.title}</em></p>
|
||||
</div>
|
||||
<script type="application/json" id="upload-init" dangerouslySetInnerHTML={{ __html: jsonForInlineScript(config) }}/>
|
||||
<script type="module" src={staticPath + '/js/client/upload.js'}></script>
|
||||
</body>
|
||||
</html>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user