will/osm-import-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Sortable per-county downloaded files table that refreshes when jobs finish, and web push notifications for job completion

Browse Source
This commit is contained in:
zyphlar
2026-10-16 20:18:39 -07:00
parent 96d5b9bac8
commit 4aca4c96fd
7 changed files with 539 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/server.py
+158 -9
View File
@@ -3,7 +3,7 @@
Flask web server for The Villages Import Tools
"""
from functools import wraps
from flask import Flask, render_template, jsonify, request, send_from_directory, session, redirect, url_for
from flask import Flask, render_template, jsonify, request, send_from_directory, session, redirect, url_for, make_response
from werkzeug.security import generate_password_hash, check_password_hash
import subprocess
import os
@@ -13,6 +13,8 @@ import shlex
import shutil
import threading
import json
import base64
import logging
from datetime import datetime, timezone
import yaml
try:
@@ -20,6 +22,13 @@ try:
_has_psutil = True
except ImportError:
_has_psutil = False
try:
from pywebpush import webpush, WebPushException
from py_vapid import Vapid
from cryptography.hazmat.primitives import serialization
_has_webpush = True
except ImportError:
_has_webpush = False
app = Flask(__name__, static_folder='static', template_folder='templates')
@@ -201,6 +210,84 @@ def _job_logs(job_id):
_load_jobs()
# ── Web push: notify subscribed browsers when a job finishes ─────────────────
# VAPID keys are generated once into /data (like the Flask secret key), or set
# VAPID_PRIVATE_KEY_FILE to use your own PEM. Browsers only allow push on HTTPS
# (or http://localhost).
VAPID_KEY_FILE = os.environ.get('VAPID_PRIVATE_KEY_FILE', '/data/.vapid_private.pem')
VAPID_SUBJECT = os.environ.get('VAPID_SUBJECT', 'mailto:admin@example.org')
PUSH_SUBSCRIPTIONS_FILE = '/data/push-subscriptions.json'
push_lock = threading.Lock()
_vapid = None
def _get_vapid():
global _vapid
if _vapid is None and _has_webpush:
if os.path.exists(VAPID_KEY_FILE):
_vapid = Vapid.from_file(VAPID_KEY_FILE)
else:
_vapid = Vapid()
_vapid.generate_keys()
_vapid.save_key(VAPID_KEY_FILE)
os.chmod(VAPID_KEY_FILE, 0o600)
return _vapid
def _vapid_public_key():
raw = _get_vapid().public_key.public_bytes(
serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint)
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
def _load_push_subscriptions():
try:
with open(PUSH_SUBSCRIPTIONS_FILE) as f:
return json.load(f)
except (OSError, ValueError):
return {}
def _save_push_subscriptions(subs):
tmp = PUSH_SUBSCRIPTIONS_FILE + '.tmp'
with open(tmp, 'w') as f:
json.dump(subs, f)
os.replace(tmp, PUSH_SUBSCRIPTIONS_FILE)
def _notify_job_finished(job):
"""Push a notification about a finished job to every subscribed browser.
Subscriptions the push service reports as gone are dropped."""
if not _has_webpush:
return
with push_lock:
subs = _load_push_subscriptions()
if not subs:
return
payload = json.dumps({'job': job})
gone = []
for endpoint, entry in subs.items():
try:
webpush(entry['subscription'], payload,
vapid_private_key=VAPID_KEY_FILE,
vapid_claims={'sub': VAPID_SUBJECT},
ttl=24 * 3600)
except WebPushException as e:
if e.response is not None and e.response.status_code in (404, 410):
gone.append(endpoint)
else:
logging.warning('Web push to %s failed: %s', endpoint[:60], e)
except Exception as e:
logging.warning('Web push to %s failed: %s', endpoint[:60], e)
if gone:
with push_lock:
subs = _load_push_subscriptions()
for endpoint in gone:
subs.pop(endpoint, None)
_save_push_subscriptions(subs)
def _cleanup_legacy_structure():
"""Remove dated folders and symlink from the old /data/YYMMDD layout,
migrating any files up to /data/{county}/ before deleting."""
@@ -231,11 +318,8 @@ def _cleanup_legacy_structure():
_cleanup_legacy_structure()
@app.route('/')
def index():
"""Main index page with script execution buttons"""
counties = get_counties()
def _list_data_files(counties):
"""Files in each county's data folder, newest first: {county_id: [{name, ts}]}."""
data_files = {}
for county in counties:
county_dir = f'/data/{county["id"]}'
@@ -244,16 +328,29 @@ def index():
for fname in sorted(os.listdir(county_dir)):
fpath = os.path.join(county_dir, fname)
if os.path.isfile(fpath):
mtime = datetime.fromtimestamp(os.path.getmtime(fpath)).strftime('%Y-%m-%d %H:%M')
files.append({'name': fname, 'mtime': mtime})
files.append({'name': fname, 'ts': int(os.path.getmtime(fpath))})
if files:
files.sort(key=lambda f: f['ts'], reverse=True)
data_files[county['id']] = files
return data_files
@app.route('/api/data-files')
def data_files_api():
return jsonify(_list_data_files(get_counties()))
@app.route('/')
def index():
"""Main index page with script execution buttons"""
counties = get_counties()
data_files = _list_data_files(counties)
import json as _json
return render_template('index.html',
counties=counties,
counties_json=_json.dumps(counties),
data_files=data_files,
data_files_json=_json.dumps(data_files),
current_user=session.get('username'))
@app.route('/map')
@@ -483,10 +580,13 @@ def run_script():
return jsonify({'error': 'Invalid script configuration'}), 400
job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
county_name = next((c['name'] for c in get_counties() if c['id'] == county), county)
job = {
'id': job_id,
'script': script_name,
'label': str(data.get('label') or script_name)[:100],
'county': county,
'county_name': county_name,
'user': session.get('username'),
'started': _utcnow(),
'ended': None,
@@ -540,6 +640,7 @@ def run_script():
_save_job(job)
# Finished logs live on disk; don't hold them in memory forever.
process_logs.pop(job_id, None)
_notify_job_finished(job)
thread = threading.Thread(target=run_command)
thread.daemon = True
@@ -592,6 +693,54 @@ def cancel_job():
except Exception as e:
return jsonify({'error': str(e)}), 500
@app.route('/sw.js')
def service_worker():
"""Served from the site root so the worker's scope covers every page."""
resp = make_response(send_from_directory(app.static_folder, 'sw.js'))
resp.headers['Content-Type'] = 'application/javascript'
resp.headers['Cache-Control'] = 'no-cache'
return resp
@app.route('/manifest.webmanifest')
def web_manifest():
"""Lets phones install the dashboard; iOS only allows web push for installed apps."""
resp = make_response(send_from_directory(app.static_folder, 'manifest.webmanifest'))
resp.headers['Content-Type'] = 'application/manifest+json'
return resp
@app.route('/api/push/key')
@require_auth
def push_key():
if not _has_webpush:
return jsonify({'error': 'pywebpush not installed on the server'}), 503
return jsonify({'publicKey': _vapid_public_key()})
@app.route('/api/push/subscribe', methods=['POST'])
@require_auth
def push_subscribe():
sub = (request.json or {}).get('subscription') or {}
keys = sub.get('keys') or {}
if not (isinstance(sub.get('endpoint'), str) and sub['endpoint'].startswith('https://')
and keys.get('p256dh') and keys.get('auth')):
return jsonify({'error': 'Invalid subscription'}), 400
with push_lock:
subs = _load_push_subscriptions()
subs[sub['endpoint']] = {'subscription': {'endpoint': sub['endpoint'], 'keys': {
'p256dh': keys['p256dh'], 'auth': keys['auth']}},
'user': session.get('username'), 'created': _utcnow()}
_save_push_subscriptions(subs)
return jsonify({'success': True})
@app.route('/api/push/unsubscribe', methods=['POST'])
@require_auth
def push_unsubscribe():
endpoint = (request.json or {}).get('endpoint')
with push_lock:
subs = _load_push_subscriptions()
if subs.pop(endpoint, None) is not None:
_save_push_subscriptions(subs)
return jsonify({'success': True})
@app.route('/api/upload-file', methods=['POST'])
@require_auth
def upload_file():