From ac34a034829e18d8e7c64b2ec69e4b10594e98a5 Mon Sep 17 00:00:00 2001 From: Alan Grainger <16197738+alangrainger@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:33:47 +1300 Subject: [PATCH] Add configurable field separator --- docs/config/upload-service.md | 39 +++++++++++++++++++++++++------ docs/tag-and-review-uploads.md | 19 ++++++++------- upload-app/config.json | 1 + upload-app/src/config.ts | 11 +++++++++ upload-app/src/filename.ts | 34 +++++++++++++++++++-------- upload-app/src/receive.ts | 5 ++-- upload-app/tests/config.test.ts | 12 +++++++++- upload-app/tests/filename.test.ts | 29 ++++++++++++++++++++--- 8 files changed, 118 insertions(+), 32 deletions(-) diff --git a/docs/config/upload-service.md b/docs/config/upload-service.md index 8a68d66..2265f5d 100644 --- a/docs/config/upload-service.md +++ b/docs/config/upload-service.md @@ -103,17 +103,42 @@ reason for the delay is explained in [When do uploads appear](/visitor-uploads#w **Type:** `bool` · **Default:** `true` Whether the upload page has an optional "Your name" field. The name goes into each stored filename between the prefix -and the original name, followed by a comma, so `IMG_0042.jpg` from Jane Smith is stored as -`ipp_upload_jane-smith,IMG_0042.jpg`. A file sent without a name is stored as `ipp_upload_IMG_0042.jpg`, with no -comma. The visitor's browser remembers the name for the next share on the same install. With `false`, the field is -not shown and a name sent anyway is ignored. +and the original name, followed by the [`fieldSeparator`](#fieldseparator), so `IMG_0042.jpg` from Jane Smith is +stored as `ipp_upload_jane-smith,IMG_0042.jpg`. A file sent without a name is stored as `ipp_upload_IMG_0042.jpg`, +with no separator. The visitor's browser remembers the name for the next share on the same install. With `false`, the +field is not shown and a name sent anyway is ignored. The name is stored as a slug: lower case, letters and digits in any script, and a hyphen for anything else, at most 32 -characters. A comma in the visitor's own filename is removed, so the comma always marks the end of the name. A -filename that already carries the prefix gets neither the prefix nor the name again. The name is the visitor's own -claim; IPP cannot check it. For what to do with it, see +characters. A filename that already carries the prefix gets neither the prefix nor the name again. The name is the +visitor's own claim; IPP cannot check it. For what to do with it, see [Find each visitor's uploads](/tag-and-review-uploads#find-each-visitors-uploads). +### `fieldSeparator` + +**Type:** `string` · **Default:** `","` + +The characters that end each visitor field in a stored filename, between the field and the original name, as in +`ipp_upload_jane-smith,IMG_0042.jpg`. The visitor's name is the only field today. The separator is removed from the +visitor's own filename, so it always marks the end of a field. A file sent with no fields has no separator at all. + +A comma reads naturally, but some people name their photos with one, as in `Venice, Italy.jpg`, and that comma is +lost. If your visitors do, set `"^"`: it is never written by a camera or a phone, and almost never by a person. + +```json +{ + "ipp": { + "upload": { + "fieldSeparator": "^" + } + } +} +``` + +Any punctuation that is legal in a filename works, with one or more characters. A letter, digit, hyphen, dot, space +or a character that is illegal in a filename (`/ ? < > \ : * | "`) is not accepted, and the default is used instead. +Changing it later leaves the files already stored with the old one. How the filename is read back is explained in +[The separator](/tag-and-review-uploads#the-separator). + ### `notifyUrl` **Type:** `string` · **Default:** `""` diff --git a/docs/tag-and-review-uploads.md b/docs/tag-and-review-uploads.md index 7bf371b..237ad44 100644 --- a/docs/tag-and-review-uploads.md +++ b/docs/tag-and-review-uploads.md @@ -39,20 +39,21 @@ The comma matters: without it, `ipp_upload_jane` would also match `jane-smith` a visitor typed, made lower case with hyphens for spaces and punctuation. A visitor who leaves the field empty is stored with the prefix alone and no comma. A visitor can type any name, so treat it as a label, not proof. -### Why a comma +### The separator -The filename is ``, where each field is followed by a comma. The separator had to be one -that a visitor's own filename never keeps, so that a stem with no comma has no fields, and otherwise the last comma +The filename is ``, where each field is followed by the separator. The separator has to be +one that a visitor's own filename never keeps, so that a stem without it has no fields, and otherwise the last one ends them. An underscore cannot do that: `ipp_upload_photo_1.jpg` would read as a visitor called `photo`. A tilde -was ruled out because Android names an edited photo `IMG_1234~2.jpg`. A comma is typeable, legal on every file -system, never written by a camera or a phone, and only rarely by a person. When a visitor's filename does contain -one, it is removed. +was ruled out because Android names an edited photo `IMG_1234~2.jpg`. The default comma is typeable, legal on every +file system, never written by a camera or a phone, and reads naturally. When a visitor's filename does contain one, +as in `Venice, Italy.jpg`, it is removed. If your visitors name their photos that way, set +[`upload.fieldSeparator`](/config/upload-service#fieldseparator) to `^`, which almost nobody types in a filename. The fields are positional so that more can be added later, such as an event code the owner hands out. A new field -goes on the end, an empty one is written empty, and a file with no fields at all has no comma. So a future +goes on the end, an empty one is written empty, and a file with no fields at all has no separator. So a future `ipp_upload_jane-smith,34,IMG_0042.jpg` and `ipp_upload_,34,IMG_0042.jpg` would sit beside today's names, and a -script that splits on the comma reads all of them: the last piece is the original filename, the rest are the fields -in order. +script that splits on the separator reads all of them: the last piece is the original filename, the rest are the +fields in order. ## Review uploads before they appear diff --git a/upload-app/config.json b/upload-app/config.json index daf1e17..c739f1e 100644 --- a/upload-app/config.json +++ b/upload-app/config.json @@ -9,6 +9,7 @@ "filenamePrefix": "ipp_upload_", "notice": "Uploaded photos may not appear in the gallery immediately.", "askName": true, + "fieldSeparator": ",", "notifyUrl": "", "notifyHeaders": {}, "notifyTimeout": 10000 diff --git a/upload-app/src/config.ts b/upload-app/src/config.ts index 2ad1774..f0e1721 100644 --- a/upload-app/src/config.ts +++ b/upload-app/src/config.ts @@ -1,4 +1,5 @@ import { getConfigOption, getNumericConfigOption } from '@ipp/core' +import { DEFAULT_FIELD_SEPARATOR, isUsableSeparator } from './filename' const MB = 1024 * 1024 @@ -37,6 +38,16 @@ export function askName (): boolean { return getConfigOption('ipp.upload.askName', true) !== false } +/** + * The string that ends each visitor field in a stored filename, from + * `ipp.upload.fieldSeparator`. The default when the configured value could be + * part of a field or a filename, see `isUsableSeparator`. + */ +export function fieldSeparator (): string { + const configured = getConfigOption('ipp.upload.fieldSeparator', DEFAULT_FIELD_SEPARATOR) + return isUsableSeparator(configured) ? configured : DEFAULT_FIELD_SEPARATOR +} + /** Text shown under the placeholder on the upload page, from `ipp.upload.notice`. Empty hides it. */ export function uploadNotice (): string { const configured = getConfigOption('ipp.upload.notice', 'Uploaded photos may not appear in the gallery immediately.') diff --git a/upload-app/src/filename.ts b/upload-app/src/filename.ts index 28a64b2..351297a 100644 --- a/upload-app/src/filename.ts +++ b/upload-app/src/filename.ts @@ -13,19 +13,33 @@ const MAX_UPLOADER_LENGTH = 32 /* A stored name is `
`. The header is the visitor - fields, each a slug, each followed by a comma: `ipp_upload_jane-smith,IMG_0042.jpg`. + fields, each a slug, each followed by the separator: `ipp_upload_jane-smith,IMG_0042.jpg`. Fields are positional, so a field added later goes on the end and an empty one is written empty: `ipp_upload_,34,IMG_0042.jpg`. When every field is empty there is no header at all, and the name is as it was before fields - existed. The visitor's own filename never keeps a comma, so a stem with no - comma has no fields and otherwise the last comma ends the header. + existed. The visitor's own filename never keeps the separator, so a stem + with no separator has no fields and otherwise the last one ends the header. */ -const FIELD_SEPARATOR = ',' + +/** The default separator between the visitor fields and the original filename, from `ipp.upload.fieldSeparator`. */ +export const DEFAULT_FIELD_SEPARATOR = ',' + +/** + * Whether `separator` can mark the end of a field: a string that `sanitize` + * leaves alone, with no slug character, dot or whitespace in it. A letter, + * digit or hyphen could be part of the name, a dot could be read as the + * extension, and whitespace would be trimmed from the ends of a stem. + */ +export function isUsableSeparator (separator: unknown): separator is string { + if (typeof separator !== 'string' || separator === '') return false + if (sanitize(separator) !== separator) return false + return !/[\p{L}\p{N}\-.\s]/u.test(separator) +} /** * The visitor's name as a filename slug: lower case, letters and digits in any * script, runs of anything else become one hyphen. Empty when nothing usable - * is left. A slug never contains the field separator. + * is left. A slug never contains a separator. */ export function uploaderSlug (raw: unknown): string { if (typeof raw !== 'string') return '' @@ -39,9 +53,9 @@ export function uploaderSlug (raw: unknown): string { } /** The header for these fields, in order, or empty when none has a value. */ -function fieldHeader (fields: string[]): string { +function fieldHeader (fields: string[], separator: string): string { if (!fields.some(Boolean)) return '' - return fields.map(field => field + FIELD_SEPARATOR).join('') + return fields.map(field => field + separator).join('') } /** @@ -51,7 +65,7 @@ function fieldHeader (fields: string[]): string { * is left, or when there is no extension: Immich types an asset by its * extension and would refuse it. */ -export function uploadFilename (raw: unknown, prefix: string, uploader = ''): string | undefined { +export function uploadFilename (raw: unknown, prefix: string, uploader = '', separator = DEFAULT_FIELD_SEPARATOR): string | undefined { if (typeof raw !== 'string') return undefined const base = (raw.normalize('NFC').replace(formatCharsRe, '').split(/[\\/]/).pop() || '') // Split the extension off first: `sanitize` truncates at 254 characters, @@ -60,14 +74,14 @@ export function uploadFilename (raw: unknown, prefix: string, uploader = ''): st if (!parts) return undefined const extension = parts[2] // The separator is removed from the visitor's name so it only ever marks a field - const stem = sanitize(parts[1]).split(FIELD_SEPARATOR).join('').trim() + const stem = sanitize(parts[1]).split(separator).join('').trim() // An empty stem is a dotfile; a leading dot is a traversal remnant. if (!stem || stem.startsWith('.')) return undefined const cleanPrefix = sanitize(prefix).trim() // Don't stack the prefix, or the name, on a file that already carries the prefix const hasPrefix = cleanPrefix !== '' && stem.startsWith(cleanPrefix) - const wanted = hasPrefix ? '' : cleanPrefix + fieldHeader([uploader]) + const wanted = hasPrefix ? '' : cleanPrefix + fieldHeader([uploader], separator) const room = MAX_FILENAME_LENGTH - wanted.length - extension.length return wanted + stem.slice(0, Math.max(1, room)) + extension } diff --git a/upload-app/src/receive.ts b/upload-app/src/receive.ts index c4cd579..ff13f1b 100644 --- a/upload-app/src/receive.ts +++ b/upload-app/src/receive.ts @@ -2,7 +2,7 @@ import { pipeline } from 'stream' import { Request, Response } from 'express-serve-static-core' import dayjs from 'dayjs' import { abortOnClose, KeyType, log, respondToInvalidRequest, SharedLink } from '@ipp/core' -import { askName, byteBudget, filenamePrefix, maxFileSize, rateLimit } from './config' +import { askName, byteBudget, fieldSeparator, filenamePrefix, maxFileSize, rateLimit } from './config' import { uploaderSlug, uploadFilename } from './filename' import { forwardUpload } from './forward' import { authoriseUpload } from './gate' @@ -67,6 +67,7 @@ export function createUploadReceiver () { const maxBytes = maxFileSize() const prefix = filenamePrefix() const withName = askName() + const separator = fieldSeparator() return async function receiveUpload (req: Request, res: Response, link: SharedLink, keyType: KeyType): Promise { const key = req.params.key @@ -93,7 +94,7 @@ export function createUploadReceiver () { // The name is the visitor's claim, kept only as part of the filename const uploader = withName ? uploaderSlug(req.query.uploader) : '' - const filename = uploadFilename(req.query.filename, prefix, uploader) + const filename = uploadFilename(req.query.filename, prefix, uploader, separator) if (!filename) { refuse(res, 400, 'Missing or unusable filename') return diff --git a/upload-app/tests/config.test.ts b/upload-app/tests/config.test.ts index f321350..dd8331f 100644 --- a/upload-app/tests/config.test.ts +++ b/upload-app/tests/config.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, afterEach } from 'vitest' -import { askName, byteBudget, filenamePrefix, maxFileSize, notifyConfig, uploadNotice } from '../src/config' +import { askName, byteBudget, fieldSeparator, filenamePrefix, maxFileSize, notifyConfig, uploadNotice } from '../src/config' import { resetConfig, setConfig } from './helpers' afterEach(resetConfig) @@ -14,6 +14,16 @@ describe('upload options', () => { expect(notifyConfig()).toBeUndefined() expect(uploadNotice()).toBe('Uploaded photos may not appear in the gallery immediately.') expect(askName()).toBe(true) + expect(fieldSeparator()).toBe(',') + }) + + it('takes a usable field separator, and falls back for one that could be part of a field', () => { + setConfig({ ipp: { upload: { fieldSeparator: '^' } } }) + expect(fieldSeparator()).toBe('^') + for (const bad of ['', '-', 'a', '.', ' ', '/', 7, null]) { + setConfig({ ipp: { upload: { fieldSeparator: bad } } }) + expect(fieldSeparator()).toBe(',') + } }) it('turns the name field off only on an explicit false', () => { diff --git a/upload-app/tests/filename.test.ts b/upload-app/tests/filename.test.ts index e4e87b2..5d1af6d 100644 --- a/upload-app/tests/filename.test.ts +++ b/upload-app/tests/filename.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest' -import { uploaderSlug, uploadFilename } from '../src/filename' +import { isUsableSeparator, uploaderSlug, uploadFilename } from '../src/filename' describe('uploadFilename', () => { it('keeps an ordinary filename', () => { @@ -56,9 +56,17 @@ describe('uploadFilename', () => { expect(uploadFilename('IMG_0042.jpg', 'ipp_upload_', '')).toBe('ipp_upload_IMG_0042.jpg') }) - it('removes commas from the visitor filename, so a comma only ever ends a field', () => { + it('removes the separator from the visitor filename, so it only ever ends a field', () => { expect(uploadFilename('Paris, France.jpg', 'ipp_upload_')).toBe('ipp_upload_Paris France.jpg') expect(uploadFilename('a,b,c.jpg', 'ipp_upload_', 'jane-smith')).toBe('ipp_upload_jane-smith,abc.jpg') + // Any other punctuation is an ordinary character and stays + expect(uploadFilename('best^photo.jpg', 'ipp_upload_', 'jane-smith')).toBe('ipp_upload_jane-smith,best^photo.jpg') + }) + + it('uses the configured separator, and removes that one instead', () => { + expect(uploadFilename('IMG_0042.jpg', 'ipp_upload_', 'jane-smith', '^')).toBe('ipp_upload_jane-smith^IMG_0042.jpg') + expect(uploadFilename('Paris, France^2.jpg', 'ipp_upload_', 'jane-smith', '^')).toBe('ipp_upload_jane-smith^Paris, France2.jpg') + expect(uploadFilename('IMG_0042.jpg', 'ipp_upload_', 'jane-smith', '~~')).toBe('ipp_upload_jane-smith~~IMG_0042.jpg') }) it('stays within 254 characters with the extension intact', () => { @@ -80,8 +88,9 @@ describe('uploaderSlug', () => { expect(uploaderSlug('山田 太郎')).toBe('山田-太郎') }) - it('never keeps a comma or an underscore', () => { + it('never keeps a separator, punctuation or an underscore', () => { expect(uploaderSlug('Smith, Jane')).toBe('smith-jane') + expect(uploaderSlug('jane^smith')).toBe('jane-smith') expect(uploaderSlug('jane_smith')).toBe('jane-smith') }) @@ -101,3 +110,17 @@ describe('uploaderSlug', () => { expect(uploaderSlug('a'.repeat(31) + ' b')).toBe('a'.repeat(31)) }) }) + +describe('isUsableSeparator', () => { + it('accepts punctuation that sanitize keeps', () => { + for (const separator of ['^', ',', '~', '#', '+', '^^', '~,']) expect(isUsableSeparator(separator)).toBe(true) + }) + + it('rejects anything that could be part of a name or a filename, or is not a string', () => { + for (const separator of ['', 'a', '1', '-', '.', ' ', 'x^', '^ ', '/', ':', '*', '"', '\u0000', 'ñ', '٣']) { + expect(isUsableSeparator(separator)).toBe(false) + } + expect(isUsableSeparator(undefined)).toBe(false) + expect(isUsableSeparator(1)).toBe(false) + }) +})