diff --git a/web/server.py b/web/server.py index b111a8b..1f7c67a 100644 --- a/web/server.py +++ b/web/server.py @@ -13,7 +13,7 @@ import shlex import shutil import threading import json -from datetime import datetime +from datetime import datetime, timezone import yaml try: import psutil @@ -120,10 +120,85 @@ def auth_status(): username = session.get('username') return jsonify({'authenticated': bool(username), 'username': username or ''}) -# ── Store running processes ──────────────────────────────────────────────────── +# ── Job registry ────────────────────────────────────────────────────────────── +# Every run is recorded under /data/jobs (metadata JSON + log file) so any +# logged-in user, on any device, can see what ran, when, and how it ended - +# even after the job finished or the server restarted. + +JOBS_DIR = '/data/jobs' +JOBS_KEEP = 200 running_processes = {} -process_logs = {} +jobs = {} # job_id -> metadata dict (see _new_job) +process_logs = {} # job_id -> list of log lines +jobs_lock = threading.Lock() + + +def _utcnow(): + return datetime.now(timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z') + + +def _job_meta_path(job_id): + return os.path.join(JOBS_DIR, f'{job_id}.json') + + +def _job_log_path(job_id): + return os.path.join(JOBS_DIR, f'{job_id}.log') + + +def _save_job(job): + os.makedirs(JOBS_DIR, exist_ok=True) + tmp = _job_meta_path(job['id']) + '.tmp' + with open(tmp, 'w') as f: + json.dump(job, f) + os.replace(tmp, _job_meta_path(job['id'])) + + +def _load_jobs(): + """Load job history from disk. Jobs left 'running' by a previous server + process can no longer be watched, so they are marked interrupted.""" + if not os.path.isdir(JOBS_DIR): + return + for fname in os.listdir(JOBS_DIR): + if not fname.endswith('.json'): + continue + try: + with open(os.path.join(JOBS_DIR, fname)) as f: + job = json.load(f) + except (OSError, ValueError): + continue + if job.get('status') == 'running': + job['status'] = 'interrupted' + job['ended'] = job.get('ended') or _utcnow() + _save_job(job) + jobs[job['id']] = job + + +def _prune_jobs(): + """Keep only the newest JOBS_KEEP jobs on disk and in memory.""" + finished = sorted((j for j in jobs.values() if j['status'] != 'running'), + key=lambda j: j['started']) + for job in finished[:max(0, len(jobs) - JOBS_KEEP)]: + jobs.pop(job['id'], None) + process_logs.pop(job['id'], None) + for path in (_job_meta_path(job['id']), _job_log_path(job['id'])): + try: + os.remove(path) + except OSError: + pass + + +def _job_logs(job_id): + if job_id in process_logs: + return process_logs[job_id] + try: + with open(_job_log_path(job_id)) as f: + return f.readlines() + except OSError: + return [] + + +_load_jobs() def _cleanup_legacy_structure(): @@ -407,9 +482,31 @@ def run_script(): else: return jsonify({'error': 'Invalid script configuration'}), 400 - job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}" + job = { + 'id': job_id, + 'script': script_name, + 'county': county, + 'user': session.get('username'), + 'started': _utcnow(), + 'ended': None, + 'status': 'running', + 'exit_code': None, + } + with jobs_lock: + jobs[job_id] = job + process_logs[job_id] = [] + _save_job(job) + _prune_jobs() def run_command(): + log_file = open(_job_log_path(job_id), 'w') + + def log(line): + process_logs[job_id].append(line) + log_file.write(line) + log_file.flush() + try: process = subprocess.Popen( cmd, @@ -419,32 +516,62 @@ def run_script(): cwd=os.path.dirname(os.path.dirname(__file__)) ) running_processes[job_id] = process - process_logs[job_id] = [] for line in process.stdout: - process_logs[job_id].append(line) + log(line) process.wait() - process_logs[job_id].append(f"\n[Process completed with exit code {process.returncode}]") + job['exit_code'] = process.returncode + if job.get('cancelled'): + job['status'] = 'cancelled' + log(f"\n[Job cancelled by {job['cancelled']}]") + else: + job['status'] = 'success' if process.returncode == 0 else 'failed' + log(f"\n[Process completed with exit code {process.returncode}]") except Exception as e: - process_logs[job_id].append(f"\n[ERROR: {str(e)}]") + job['status'] = 'failed' + log(f"\n[ERROR: {str(e)}]") finally: - if job_id in running_processes: - del running_processes[job_id] + running_processes.pop(job_id, None) + log_file.close() + with jobs_lock: + job['ended'] = _utcnow() + _save_job(job) + # Finished logs live on disk; don't hold them in memory forever. + process_logs.pop(job_id, None) thread = threading.Thread(target=run_command) thread.daemon = True thread.start() - return jsonify({'job_id': job_id, 'status': 'started'}) + return jsonify({'job_id': job_id, 'status': 'started', 'job': job}) + +@app.route('/api/jobs') +@require_auth +def list_jobs(): + """Most recent job overall, plus the most recent job for each script/county + pair, so every device can show the same banner and per-step status.""" + with jobs_lock: + ordered = sorted(jobs.values(), key=lambda j: j['started']) + latest_by_step = {} + for job in ordered: + latest_by_step[f"{job['script']}|{job['county']}"] = job + return jsonify({ + 'latest': ordered[-1] if ordered else None, + 'steps': latest_by_step, + 'running': [j['id'] for j in ordered if j['status'] == 'running'], + }) @app.route('/api/job-status/') +@require_auth def job_status(job_id): """Get status and logs for a job""" - is_running = job_id in running_processes - logs = process_logs.get(job_id, []) - return jsonify({'job_id': job_id, 'running': is_running, 'logs': logs}) + job = jobs.get(job_id) + if not job: + return jsonify({'error': 'Job not found'}), 404 + return jsonify({'job_id': job_id, 'running': job['status'] == 'running', + 'job': job, 'logs': _job_logs(job_id)}) @app.route('/api/cancel-job', methods=['POST']) @require_auth @@ -459,10 +586,8 @@ def cancel_job(): return jsonify({'error': 'Job not found or already completed'}), 404 try: - process = running_processes[job_id] - process.terminate() - process_logs[job_id].append('\n[Job cancelled by user]') - del running_processes[job_id] + jobs[job_id]['cancelled'] = session.get('username') or 'user' + running_processes[job_id].terminate() return jsonify({'success': True}) except Exception as e: return jsonify({'error': str(e)}), 500 diff --git a/web/templates/index.html b/web/templates/index.html index d42a0ad..7076614 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -166,6 +166,29 @@ color: #444; } + .step-name { display: block; } + + /* Last run of this step (for the selected county); click to view its output */ + .step-last { + display: block; + margin-top: 2px; + padding: 0; + border: none; + background: none; + font: inherit; + font-size: 10px; + color: #888; + text-align: left; + cursor: pointer; + } + .step-last:hover { text-decoration: underline; } + .step-last[hidden] { display: none; } + .step-last.running { color: #1f78b4; } + .step-last.success { color: #2e7d32; } + .step-last.failed { color: #c62828; } + .step-last.cancelled, + .step-last.interrupted { color: #b26a00; } + .step-actions { display: flex; align-items: center; @@ -296,6 +319,27 @@ display: none; } + /* Banner for the job whose output is shown; persists after it ends */ + #jobBanner { + padding: 7px 14px; + font-size: 12px; + flex-shrink: 0; + display: none; + background: #1e2e3a; + color: #70b8e0; + border-bottom: 1px solid #111; + } + #jobBanner .job-title { font-weight: 700; } + #jobBanner .job-meta { display: block; margin-top: 2px; font-size: 11px; opacity: 0.85; } + #jobBanner .job-back { + margin-left: 6px; padding: 0; border: none; background: none; + color: inherit; font: inherit; text-decoration: underline; cursor: pointer; + } + #jobBanner.success { background: #1e3a24; color: #6fcf7f; } + #jobBanner.failed { background: #3a1e1e; color: #e07070; } + #jobBanner.cancelled, + #jobBanner.interrupted { background: #3a301e; color: #e0b070; } + #status.success { background: #1e3a24; color: #6fcf7f; } #status.error { background: #3a1e1e; color: #e07070; } #status.info { background: #1e2e3a; color: #70b8e0; } @@ -507,30 +551,30 @@
Roads
-
+
1
-
Download county data
+
Download county data
-
+
2
-
Download OSM roads
+
Download OSM roads
-
+
3
-
Convert to GeoJSON
+
Convert to GeoJSON
-
+
4
-
Diff vs OSM
+
Diff vs OSM
@@ -540,30 +584,30 @@
Paths
-
+
1
-
Upload county paths
+
Upload county paths
-
+
2
-
Download OSM paths
+
Download OSM paths
-
+
3
-
Convert to GeoJSON
+
Convert to GeoJSON
-
+
4
-
Diff vs OSM
+
Diff vs OSM
@@ -573,30 +617,30 @@
Addresses
-
+
1
-
Download county data
+
Download county data
-
+
2
-
Download OSM addresses
+
Download OSM addresses
-
+
3
-
Convert to GeoJSON
+
Convert to GeoJSON
-
+
4
-
Diff vs OSM
+
Diff vs OSM
-
+
5
-
Generate City Boundaries Report
+
Generate City Boundaries Report
@@ -642,6 +686,7 @@ Output
+
@@ -670,8 +715,12 @@ const COUNTY_CONFIG = {{ counties_json | safe }}; const IS_AUTHENTICATED = {{ 'true' if current_user else 'false' }}; let activeCounty = COUNTY_CONFIG.length ? COUNTY_CONFIG[0].id : ''; - let currentJobId = null; - let logCheckInterval = null; + let jobsData = null; // last /api/jobs response + let viewedJob = null; // job whose output is shown in the console + let followLatest = true; // switch the console to new jobs as they start + let lastSeenLatestId = null; + let jobsTimer = null; + let uploading = false; function requireLogin() { showStatus('Not authenticated – please log in.', 'error'); @@ -693,6 +742,8 @@ panel.style.setProperty('--active-color', config ? config.color : '#28a745'); renderPathsActions(county, config); + renderStepStatus(); + if (jobsData && jobsData.running.length) disableButtons(); } function renderPathsActions(county, config) { @@ -738,11 +789,8 @@ /* ── Script runner ── */ function runScript(scriptName) { if (!IS_AUTHENTICATED) return requireLogin(); - const logsEl = document.getElementById('logs'); - logsEl.textContent = ''; - showStatus(`Running: ${scriptName} (${activeCounty})`, 'info'); + hideStatus(); disableButtons(); - document.getElementById('cancelButton').classList.add('active'); // On narrow screens the console is below the steps; bring it into view if (window.matchMedia('(max-width: 768px)').matches) { document.querySelector('.right').scrollIntoView({ behavior: 'smooth', block: 'start' }); @@ -757,68 +805,185 @@ fetch('/api/run-script', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ script: scriptName, county: activeCounty, forceDownload }) + body: JSON.stringify({ script: scriptName, county: scriptName === 'ls' ? '' : activeCounty, forceDownload }) }) .then(r => r.json()) .then(data => { if (data.error) { showStatus(`Error: ${data.error}`, 'error'); enableButtons(); - document.getElementById('cancelButton').classList.remove('active'); return; } - currentJobId = data.job_id; - if (logCheckInterval) clearInterval(logCheckInterval); - logCheckInterval = setInterval(checkJobStatus, 1000); - startStatsPolling(); + followLatest = true; + lastSeenLatestId = data.job_id; + viewJob(data.job_id, true); }) .catch(err => { showStatus(`Error: ${err.message}`, 'error'); enableButtons(); - document.getElementById('cancelButton').classList.remove('active'); }); } function cancelJob() { - if (!currentJobId || !confirm('Cancel the running job?')) return; + if (!viewedJob || viewedJob.status !== 'running' || !confirm('Cancel the running job?')) return; fetch('/api/cancel-job', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ job_id: currentJobId }) + body: JSON.stringify({ job_id: viewedJob.id }) }) .then(r => r.json()) .then(data => { - if (data.success) { - showStatus('Cancelled.', 'info'); - clearInterval(logCheckInterval); - enableButtons(); - document.getElementById('cancelButton').classList.remove('active'); - currentJobId = null; - stopStatsPolling(); - } else { - showStatus(`Cancel failed: ${data.error}`, 'error'); - } + if (data.success) refreshJobs(); + else showStatus(`Cancel failed: ${data.error}`, 'error'); }); } - function checkJobStatus() { - if (!currentJobId) return; - fetch(`/api/job-status/${currentJobId}`) + /* ── Shared job status (same on every device) ── */ + function jobLabel(job) { + if (job.script === 'ls') return 'List files'; + const step = document.querySelector(`.step[data-script="${job.script}"]`); + if (!step) return job.script; + const group = step.closest('.wf-group').querySelector('.wf-title').textContent.trim(); + return `${group} · ${step.querySelector('.step-name').textContent.trim()}`; + } + + function countyName(id) { + const c = COUNTY_CONFIG.find(c => c.id === id); + return c ? c.name : id; + } + + function fmtTime(iso) { + if (!iso) return ''; + const d = new Date(iso); + const time = d.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }); + if (d.toDateString() === new Date().toDateString()) return time; + return `${d.toLocaleDateString([], { month: 'short', day: 'numeric' })} ${time}`; + } + + function fmtDuration(fromIso, toIso) { + let s = Math.max(0, Math.round(((toIso ? new Date(toIso) : new Date()) - new Date(fromIso)) / 1000)); + const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60); + s %= 60; + return h ? `${h}h ${m}m` : m ? `${m}m ${s}s` : `${s}s`; + } + + const STATUS_TEXT = { + running: 'Running', success: 'Succeeded', failed: 'Failed', + cancelled: 'Cancelled', interrupted: 'Interrupted (server restarted)' + }; + const STATUS_ICON = { running: '⟳', success: '✓', failed: '✗', cancelled: '⊘', interrupted: '⚠' }; + + function statusText(job) { + let t = STATUS_TEXT[job.status] || job.status; + if (job.status === 'failed' && job.exit_code !== null && job.exit_code !== undefined) t += ` (exit ${job.exit_code})`; + return t; + } + + function renderBanner() { + const el = document.getElementById('jobBanner'); + const job = viewedJob; + if (!job) { el.style.display = 'none'; return; } + const county = job.county ? ` (${countyName(job.county)})` : ''; + const by = job.user ? ` by ${job.user}` : ''; + const meta = job.status === 'running' + ? `Started ${fmtTime(job.started)}${by} · ${fmtDuration(job.started)} elapsed` + : `Started ${fmtTime(job.started)}${by} · ended ${fmtTime(job.ended)} · took ${fmtDuration(job.started, job.ended)}`; + el.className = job.status; + el.innerHTML = ''; + const title = document.createElement('span'); + title.className = 'job-title'; + title.textContent = `${STATUS_ICON[job.status] || ''} ${statusText(job)}: ${jobLabel(job)}${county}`; + el.appendChild(title); + if (!followLatest && jobsData && jobsData.latest && jobsData.latest.id !== job.id) { + const back = document.createElement('button'); + back.className = 'job-back'; + back.textContent = 'show latest'; + back.onclick = () => { followLatest = true; viewJob(jobsData.latest.id); }; + el.appendChild(back); + } + const m = document.createElement('span'); + m.className = 'job-meta'; + m.textContent = meta; + el.appendChild(m); + el.style.display = 'block'; + document.getElementById('cancelButton').classList.toggle('active', job.status === 'running'); + } + + function renderStepStatus() { + document.querySelectorAll('.step[data-script]').forEach(step => { + const el = step.querySelector('.step-last'); + const job = jobsData && jobsData.steps[`${step.dataset.script}|${activeCounty}`]; + if (!job) { el.hidden = true; return; } + el.hidden = false; + el.className = `step-last ${job.status}`; + el.textContent = job.status === 'running' + ? `${STATUS_ICON.running} Running · started ${fmtTime(job.started)}` + : `${STATUS_ICON[job.status] || ''} ${statusText(job)} · ${fmtTime(job.ended)}`; + el.title = `Started ${new Date(job.started).toLocaleString()}` + + (job.ended ? `, ended ${new Date(job.ended).toLocaleString()}` : '') + + (job.user ? ` by ${job.user}` : '') + ' — click to view output'; + el.onclick = () => { followLatest = false; viewJob(job.id, true); }; + }); + } + + // Load a job's output into the console and make it the banner subject. + function viewJob(jobId, scroll) { + return fetch(`/api/job-status/${encodeURIComponent(jobId)}`) .then(r => r.json()) .then(data => { + if (data.error) return; + const wasRunning = viewedJob && viewedJob.id === data.job.id && viewedJob.status === 'running'; + viewedJob = data.job; const logsEl = document.getElementById('logs'); + const atBottom = logsEl.scrollTop + logsEl.clientHeight >= logsEl.scrollHeight - 20; logsEl.textContent = data.logs.join(''); - logsEl.scrollTop = logsEl.scrollHeight; - if (!data.running) { - clearInterval(logCheckInterval); - showStatus('Done.', 'success'); - enableButtons(); - document.getElementById('cancelButton').classList.remove('active'); - currentJobId = null; - stopStatsPolling(); - } + if (scroll || atBottom) logsEl.scrollTop = logsEl.scrollHeight; + if (data.job.status === 'running') startStatsPolling(); + else if (wasRunning || statsInterval) stopStatsPolling(); + renderBanner(); + scheduleJobsRefresh(); }) - .catch(err => console.error('Status check error:', err)); + .catch(err => console.error('Job status error:', err)); + } + + function refreshJobs() { + return fetch('/api/jobs') + .then(r => r.json()) + .then(data => { + if (data.error) return; + jobsData = data; + renderStepStatus(); + if (data.running.length) disableButtons(); else if (!uploading) enableButtons(); + const latest = data.latest; + // A newly started job (from any device) takes over the console + if (latest && latest.id !== lastSeenLatestId) { + lastSeenLatestId = latest.id; + followLatest = true; + } + const target = followLatest && latest ? latest.id : viewedJob && viewedJob.id; + const switching = target && (!viewedJob || viewedJob.id !== target); + if (switching || (viewedJob && viewedJob.status === 'running')) { + return viewJob(target, switching); + } + renderBanner(); + }) + .catch(err => console.error('Jobs refresh error:', err)) + .finally(scheduleJobsRefresh); + } + + // Poll quickly while something is running, slowly otherwise, so jobs + // started on another device show up here too. + function scheduleJobsRefresh() { + clearTimeout(jobsTimer); + const busy = (viewedJob && viewedJob.status === 'running') || (jobsData && jobsData.running.length); + jobsTimer = setTimeout(refreshJobs, busy ? 1000 : 5000); + } + + if (IS_AUTHENTICATED) { + refreshJobs(); + document.addEventListener('visibilitychange', () => { if (!document.hidden) refreshJobs(); }); + } else { + document.getElementById('logs').innerHTML = 'Log in to see job output.'; } function uploadFile(inputId, dest) { @@ -834,6 +999,7 @@ formData.append('dest', dest); showStatus(`Uploading ${file.name}...`, 'info'); + uploading = true; disableButtons(); fetch('/api/upload-file', { method: 'POST', body: formData }) @@ -847,7 +1013,7 @@ } }) .catch(err => showStatus(`Error: ${err.message}`, 'error')) - .finally(() => enableButtons()); + .finally(() => { uploading = false; refreshJobs(); }); } function clearData() { @@ -873,6 +1039,10 @@ el.style.display = 'block'; } + function hideStatus() { + document.getElementById('status').style.display = 'none'; + } + function disableButtons() { document.querySelectorAll('.btn-run').forEach(b => b.disabled = true); }