zyphlar
1100 lines
41 KiB
Python
1100 lines
41 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Flask web server for The Villages Import Tools
|
|
"""
|
|
from functools import wraps
|
|
from flask import Flask, render_template, jsonify, request, send_from_directory, send_file, session, redirect, url_for, make_response
|
|
from werkzeug.utils import safe_join
|
|
from werkzeug.security import generate_password_hash, check_password_hash
|
|
import subprocess
|
|
import os
|
|
import re
|
|
import secrets
|
|
import shlex
|
|
import shutil
|
|
import threading
|
|
import json
|
|
import base64
|
|
import gzip
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
import yaml
|
|
try:
|
|
import psutil
|
|
_has_psutil = True
|
|
except ImportError:
|
|
_has_psutil = False
|
|
try:
|
|
from pywebpush import webpush, WebPushException
|
|
from py_vapid import Vapid
|
|
from cryptography.hazmat.primitives import serialization
|
|
_has_webpush = True
|
|
except ImportError:
|
|
_has_webpush = False
|
|
|
|
app = Flask(__name__, static_folder='static', template_folder='templates')
|
|
|
|
# ── Secret key ────────────────────────────────────────────────────────────────
|
|
|
|
def _get_or_create_secret_key():
|
|
key = os.environ.get('FLASK_SECRET_KEY', '')
|
|
if key:
|
|
return key
|
|
key_file = '/data/.secret_key'
|
|
if os.path.exists(key_file):
|
|
with open(key_file) as f:
|
|
return f.read().strip()
|
|
key = secrets.token_hex(32)
|
|
try:
|
|
os.makedirs('/data', exist_ok=True)
|
|
with open(key_file, 'w') as f:
|
|
f.write(key)
|
|
except Exception:
|
|
pass
|
|
return key
|
|
|
|
app.secret_key = _get_or_create_secret_key()
|
|
|
|
# ── Users / auth ───────────────────────────────────────────────────────────────
|
|
|
|
USERS_FILE = '/data/users.yml'
|
|
|
|
|
|
def _load_users():
|
|
try:
|
|
with open(USERS_FILE) as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return {u['username']: u['password_hash'] for u in data.get('users', [])}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _save_users(users_dict):
|
|
users_list = [{'username': u, 'password_hash': h} for u, h in sorted(users_dict.items())]
|
|
tmp = USERS_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
yaml.dump({'users': users_list}, f, default_flow_style=False, allow_unicode=True)
|
|
os.replace(tmp, USERS_FILE)
|
|
|
|
|
|
def _seed_admin():
|
|
"""Create or update the admin user from the ADMIN_PASSWORD env var."""
|
|
password = os.environ.get('ADMIN_PASSWORD', '')
|
|
if not password:
|
|
return
|
|
users = _load_users()
|
|
users['admin'] = generate_password_hash(password)
|
|
_save_users(users)
|
|
|
|
|
|
_seed_admin()
|
|
|
|
|
|
def require_auth(f):
|
|
@wraps(f)
|
|
def decorated(*args, **kwargs):
|
|
if not session.get('username'):
|
|
if request.is_json or request.path.startswith('/api/'):
|
|
return jsonify({'error': 'Authentication required'}), 401
|
|
return redirect(url_for('login_page', next=request.path))
|
|
return f(*args, **kwargs)
|
|
return decorated
|
|
|
|
|
|
@app.route('/login', methods=['GET'])
|
|
def login_page():
|
|
if session.get('username'):
|
|
return redirect(request.args.get('next', '/'))
|
|
return render_template('login.html', next=request.args.get('next', '/'), error=None)
|
|
|
|
|
|
@app.route('/login', methods=['POST'])
|
|
def login():
|
|
username = request.form.get('username', '').strip()
|
|
password = request.form.get('password', '')
|
|
next_url = request.form.get('next', '/')
|
|
users = _load_users()
|
|
if username and username in users and check_password_hash(users[username], password):
|
|
session['username'] = username
|
|
return redirect(next_url)
|
|
return render_template('login.html', next=next_url, error='Invalid username or password')
|
|
|
|
|
|
@app.route('/logout', methods=['POST'])
|
|
def logout():
|
|
session.clear()
|
|
return redirect('/')
|
|
|
|
|
|
@app.route('/api/auth-status')
|
|
def auth_status():
|
|
username = session.get('username')
|
|
return jsonify({'authenticated': bool(username), 'username': username or ''})
|
|
|
|
# ── Job registry ──────────────────────────────────────────────────────────────
|
|
# Every run is recorded under /data/jobs (metadata JSON + log file) so any
|
|
# logged-in user, on any device, can see what ran, when, and how it ended -
|
|
# even after the job finished or the server restarted.
|
|
|
|
JOBS_DIR = '/data/jobs'
|
|
JOBS_KEEP = 200
|
|
|
|
running_processes = {}
|
|
jobs = {} # job_id -> metadata dict (see _new_job)
|
|
process_logs = {} # job_id -> list of log lines
|
|
jobs_lock = threading.Lock()
|
|
|
|
|
|
def _utcnow():
|
|
return datetime.now(timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z')
|
|
|
|
|
|
def _job_meta_path(job_id):
|
|
return os.path.join(JOBS_DIR, f'{job_id}.json')
|
|
|
|
|
|
def _job_log_path(job_id):
|
|
return os.path.join(JOBS_DIR, f'{job_id}.log')
|
|
|
|
|
|
def _save_job(job):
|
|
os.makedirs(JOBS_DIR, exist_ok=True)
|
|
tmp = _job_meta_path(job['id']) + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
json.dump(job, f)
|
|
os.replace(tmp, _job_meta_path(job['id']))
|
|
|
|
|
|
def _load_jobs():
|
|
"""Load job history from disk. Jobs left 'running' by a previous server
|
|
process can no longer be watched, so they are marked interrupted."""
|
|
if not os.path.isdir(JOBS_DIR):
|
|
return
|
|
for fname in os.listdir(JOBS_DIR):
|
|
if not fname.endswith('.json'):
|
|
continue
|
|
try:
|
|
with open(os.path.join(JOBS_DIR, fname)) as f:
|
|
job = json.load(f)
|
|
except (OSError, ValueError):
|
|
continue
|
|
if job.get('status') == 'running':
|
|
job['status'] = 'interrupted'
|
|
job['ended'] = job.get('ended') or _utcnow()
|
|
_save_job(job)
|
|
jobs[job['id']] = job
|
|
|
|
|
|
def _prune_jobs():
|
|
"""Keep only the newest JOBS_KEEP jobs on disk and in memory."""
|
|
finished = sorted((j for j in jobs.values() if j['status'] != 'running'),
|
|
key=lambda j: j['started'])
|
|
for job in finished[:max(0, len(jobs) - JOBS_KEEP)]:
|
|
jobs.pop(job['id'], None)
|
|
process_logs.pop(job['id'], None)
|
|
for path in (_job_meta_path(job['id']), _job_log_path(job['id'])):
|
|
try:
|
|
os.remove(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _job_logs(job_id):
|
|
if job_id in process_logs:
|
|
return process_logs[job_id]
|
|
try:
|
|
with open(_job_log_path(job_id)) as f:
|
|
return f.readlines()
|
|
except OSError:
|
|
return []
|
|
|
|
|
|
_load_jobs()
|
|
|
|
|
|
# ── Web push: notify subscribed browsers when a job finishes ─────────────────
|
|
# VAPID keys are generated once into /data (like the Flask secret key), or set
|
|
# VAPID_PRIVATE_KEY_FILE to use your own PEM. Browsers only allow push on HTTPS
|
|
# (or http://localhost).
|
|
|
|
VAPID_KEY_FILE = os.environ.get('VAPID_PRIVATE_KEY_FILE', '/data/.vapid_private.pem')
|
|
VAPID_SUBJECT = os.environ.get('VAPID_SUBJECT', 'mailto:admin@example.org')
|
|
PUSH_SUBSCRIPTIONS_FILE = '/data/push-subscriptions.json'
|
|
push_lock = threading.Lock()
|
|
_vapid = None
|
|
|
|
|
|
def _get_vapid():
|
|
global _vapid
|
|
if _vapid is None and _has_webpush:
|
|
if os.path.exists(VAPID_KEY_FILE):
|
|
_vapid = Vapid.from_file(VAPID_KEY_FILE)
|
|
else:
|
|
_vapid = Vapid()
|
|
_vapid.generate_keys()
|
|
_vapid.save_key(VAPID_KEY_FILE)
|
|
os.chmod(VAPID_KEY_FILE, 0o600)
|
|
return _vapid
|
|
|
|
|
|
def _vapid_public_key():
|
|
raw = _get_vapid().public_key.public_bytes(
|
|
serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint)
|
|
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
|
|
|
|
|
|
def _load_push_subscriptions():
|
|
try:
|
|
with open(PUSH_SUBSCRIPTIONS_FILE) as f:
|
|
return json.load(f)
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def _save_push_subscriptions(subs):
|
|
tmp = PUSH_SUBSCRIPTIONS_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
json.dump(subs, f)
|
|
os.replace(tmp, PUSH_SUBSCRIPTIONS_FILE)
|
|
|
|
|
|
def _notify_job_finished(job):
|
|
"""Push a notification about a finished job to every subscribed browser.
|
|
Subscriptions the push service reports as gone are dropped."""
|
|
if not _has_webpush:
|
|
return
|
|
with push_lock:
|
|
subs = _load_push_subscriptions()
|
|
if not subs:
|
|
return
|
|
payload = json.dumps({'job': job})
|
|
gone = []
|
|
for endpoint, entry in subs.items():
|
|
try:
|
|
webpush(entry['subscription'], payload,
|
|
vapid_private_key=VAPID_KEY_FILE,
|
|
vapid_claims={'sub': VAPID_SUBJECT},
|
|
ttl=24 * 3600)
|
|
except WebPushException as e:
|
|
if e.response is not None and e.response.status_code in (404, 410):
|
|
gone.append(endpoint)
|
|
else:
|
|
logging.warning('Web push to %s failed: %s', endpoint[:60], e)
|
|
except Exception as e:
|
|
logging.warning('Web push to %s failed: %s', endpoint[:60], e)
|
|
if gone:
|
|
with push_lock:
|
|
subs = _load_push_subscriptions()
|
|
for endpoint in gone:
|
|
subs.pop(endpoint, None)
|
|
_save_push_subscriptions(subs)
|
|
|
|
|
|
def _cleanup_legacy_structure():
|
|
"""Remove dated folders and symlink from the old /data/YYMMDD layout,
|
|
migrating any files up to /data/{county}/ before deleting."""
|
|
data_dir = '/data'
|
|
if not os.path.exists(data_dir):
|
|
return
|
|
|
|
symlink = os.path.join(data_dir, 'latest')
|
|
if os.path.islink(symlink):
|
|
os.unlink(symlink)
|
|
|
|
dated_pattern = re.compile(r'^\d{6}$')
|
|
for entry in os.listdir(data_dir):
|
|
entry_path = os.path.join(data_dir, entry)
|
|
if os.path.isdir(entry_path) and dated_pattern.match(entry):
|
|
for county_dir in os.listdir(entry_path):
|
|
src_county = os.path.join(entry_path, county_dir)
|
|
if os.path.isdir(src_county):
|
|
dst_county = os.path.join(data_dir, county_dir)
|
|
os.makedirs(dst_county, exist_ok=True)
|
|
for fname in os.listdir(src_county):
|
|
src_file = os.path.join(src_county, fname)
|
|
dst_file = os.path.join(dst_county, fname)
|
|
if os.path.isfile(src_file) and not os.path.exists(dst_file):
|
|
shutil.move(src_file, dst_file)
|
|
shutil.rmtree(entry_path)
|
|
|
|
_cleanup_legacy_structure()
|
|
|
|
|
|
def _list_data_files(counties):
|
|
"""Files in each county's data folder, newest first: {county_id: [{name, ts}]}."""
|
|
data_files = {}
|
|
for county in counties:
|
|
county_dir = f'/data/{county["id"]}'
|
|
if os.path.isdir(county_dir):
|
|
files = []
|
|
for fname in sorted(os.listdir(county_dir)):
|
|
fpath = os.path.join(county_dir, fname)
|
|
if os.path.isfile(fpath):
|
|
files.append({'name': fname, 'ts': int(os.path.getmtime(fpath))})
|
|
if files:
|
|
files.sort(key=lambda f: f['ts'], reverse=True)
|
|
data_files[county['id']] = files
|
|
return data_files
|
|
|
|
|
|
@app.route('/api/data-files')
|
|
def data_files_api():
|
|
return jsonify(_list_data_files(get_counties()))
|
|
|
|
|
|
@app.route('/')
|
|
def index():
|
|
"""Main index page with script execution buttons"""
|
|
counties = get_counties()
|
|
data_files = _list_data_files(counties)
|
|
|
|
import json as _json
|
|
return render_template('index.html',
|
|
counties=counties,
|
|
counties_json=_json.dumps(counties),
|
|
data_files_json=_json.dumps(data_files),
|
|
current_user=session.get('username'))
|
|
|
|
@app.route('/map')
|
|
def map_viewer():
|
|
"""Map viewer page"""
|
|
return render_template('map.html')
|
|
|
|
# ── Reports ────────────────────────────────────────────────────────────────
|
|
|
|
@app.route('/reports')
|
|
def reports_page():
|
|
"""Reports index page"""
|
|
return render_template('reports.html', current_user=session.get('username'))
|
|
|
|
@app.route('/reports/city-boundaries')
|
|
def city_boundaries_report():
|
|
"""City name agreement report: county vs OSM addr:city, on a map"""
|
|
counties = get_counties()
|
|
available = []
|
|
pinned_cities = {}
|
|
for county in counties:
|
|
path = f'/data/{county["id"]}/city-comparison.geojson'
|
|
if os.path.isfile(path):
|
|
available.append(county['id'])
|
|
pinned_cities[county['id']] = county.get('pinned_cities') or []
|
|
return render_template('city-boundaries-report.html',
|
|
counties=counties,
|
|
available_counties=available,
|
|
pinned_cities=pinned_cities,
|
|
current_user=session.get('username'))
|
|
|
|
# ── map-checker: hand-traced boundary tool ──────────────────────────────────
|
|
#
|
|
# thevillages.com's Terms of Use prohibit automated access, so this only ever
|
|
# saves what a human draws by hand while looking at the official map
|
|
# themselves - nothing here fetches from thevillages.com. See map-checker/README.md.
|
|
|
|
MAP_CHECKER_DIR = '/data/map-checker'
|
|
MAP_CHECKER_BOUNDARY_FILE = os.path.join(MAP_CHECKER_DIR, 'boundary.geojson')
|
|
MAP_CHECKER_DRAFT_FILE = os.path.join(MAP_CHECKER_DIR, 'draft.json')
|
|
|
|
@app.route('/map-checker')
|
|
@require_auth
|
|
def map_checker_page():
|
|
"""Boundary tracer: hand-trace a polygon against a reference map, save as GeoJSON.
|
|
|
|
Requires login so tracing work is never done (and lost) as an anonymous session."""
|
|
return render_template('boundary-tracer.html',
|
|
counties=get_counties(),
|
|
current_user=session.get('username'))
|
|
|
|
@app.route('/api/map-checker/boundary', methods=['GET'])
|
|
@require_auth
|
|
def get_map_checker_boundary():
|
|
if not os.path.isfile(MAP_CHECKER_BOUNDARY_FILE):
|
|
return jsonify({'type': 'FeatureCollection', 'features': []})
|
|
with open(MAP_CHECKER_BOUNDARY_FILE) as f:
|
|
return jsonify(json.load(f))
|
|
|
|
@app.route('/api/map-checker/boundary', methods=['POST'])
|
|
@require_auth
|
|
def save_map_checker_boundary():
|
|
data = request.json or {}
|
|
if data.get('type') != 'FeatureCollection' or not isinstance(data.get('features'), list):
|
|
return jsonify({'error': 'Expected a GeoJSON FeatureCollection'}), 400
|
|
for feat in data['features']:
|
|
geom_type = ((feat.get('geometry') or {}).get('type'))
|
|
if geom_type not in ('Polygon', 'MultiPolygon'):
|
|
return jsonify({'error': f'Unsupported geometry type: {geom_type}'}), 400
|
|
os.makedirs(MAP_CHECKER_DIR, exist_ok=True)
|
|
tmp = MAP_CHECKER_BOUNDARY_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
json.dump(data, f)
|
|
os.replace(tmp, MAP_CHECKER_BOUNDARY_FILE)
|
|
return jsonify({'success': True, 'features': len(data['features'])})
|
|
|
|
|
|
def _is_latlng_list(val):
|
|
if not isinstance(val, list):
|
|
return False
|
|
for pt in val:
|
|
if not (isinstance(pt, list) and len(pt) == 2 and
|
|
all(isinstance(c, (int, float)) for c in pt)):
|
|
return False
|
|
return True
|
|
|
|
|
|
@app.route('/api/map-checker/draft', methods=['GET'])
|
|
@require_auth
|
|
def get_map_checker_draft():
|
|
if not os.path.isfile(MAP_CHECKER_DRAFT_FILE):
|
|
return jsonify({'polygons': [], 'inProgress': []})
|
|
with open(MAP_CHECKER_DRAFT_FILE) as f:
|
|
return jsonify(json.load(f))
|
|
|
|
@app.route('/api/map-checker/draft', methods=['POST'])
|
|
@require_auth
|
|
def save_map_checker_draft():
|
|
"""Autosave endpoint hit after every vertex/edit so in-progress tracing
|
|
survives a crashed tab or dropped connection, even before the user
|
|
explicitly finishes a polygon or clicks Save to server."""
|
|
data = request.json or {}
|
|
polygons = data.get('polygons')
|
|
in_progress = data.get('inProgress', [])
|
|
if not isinstance(polygons, list):
|
|
return jsonify({'error': 'Expected a polygons list'}), 400
|
|
for p in polygons:
|
|
if not (isinstance(p, dict) and isinstance(p.get('name'), str) and _is_latlng_list(p.get('latlngs'))):
|
|
return jsonify({'error': 'Malformed polygon entry'}), 400
|
|
if not _is_latlng_list(in_progress):
|
|
return jsonify({'error': 'Malformed inProgress point list'}), 400
|
|
os.makedirs(MAP_CHECKER_DIR, exist_ok=True)
|
|
tmp = MAP_CHECKER_DRAFT_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
json.dump({'polygons': polygons, 'inProgress': in_progress}, f)
|
|
os.replace(tmp, MAP_CHECKER_DRAFT_FILE)
|
|
return jsonify({'success': True})
|
|
|
|
def get_script_map():
|
|
"""Build script map dynamically from county configuration."""
|
|
counties = get_counties()
|
|
|
|
sm = {
|
|
'ls': 'ls -alR /data',
|
|
}
|
|
|
|
dl_roads = {}; dl_addrs = {}; dl_paths = {}
|
|
osm_roads = {}; osm_addrs = {}; osm_paths = {}
|
|
conv_paths = {}
|
|
diff_roads = {}; diff_addrs = {}; diff_paths = {}
|
|
city_report = {}
|
|
|
|
for c in counties:
|
|
cid = c['id']
|
|
name = c['name']
|
|
state = c['state']
|
|
base = f'/data/{cid}'
|
|
|
|
# Each county download also converts the shapefile to GeoJSON, so the
|
|
# diff can run straight after it. Uploaded paths are converted by a
|
|
# separate convert-paths job that the dashboard starts after the upload.
|
|
conv_roads = f'python shp-to-geojson.py {base}/roads.shp.zip {base}/county-roads.geojson'
|
|
conv_addrs = f'python convert-addresses.py {base}/addresses.shp.zip {base}/county-addresses.geojson'
|
|
conv_paths_cmd = f'python shp-to-geojson.py {base}/paths.shp.zip {base}/county-paths.geojson'
|
|
|
|
if c.get('roads_url'):
|
|
dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip && {conv_roads}'
|
|
if c.get('addresses_url'):
|
|
dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip && {conv_addrs}'
|
|
pu = c.get('paths_url', '')
|
|
if pu == 'upload':
|
|
dl_paths[cid] = {'type': 'upload', 'dest': f'{base}/paths.shp.zip'}
|
|
conv_paths[cid] = conv_paths_cmd
|
|
elif pu:
|
|
dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip && {conv_paths_cmd}'
|
|
|
|
osm_roads[cid] = ['python', 'download-overpass.py', '--type', 'highways', name, state, f'{base}/osm-roads.geojson']
|
|
osm_addrs[cid] = ['python', 'download-overpass.py', '--type', 'addresses', name, state, f'{base}/osm-addresses.geojson']
|
|
if pu:
|
|
osm_paths[cid] = ['python', 'download-overpass.py', '--type', 'paths', name, state, f'{base}/osm-paths.geojson']
|
|
|
|
diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson']
|
|
diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base]
|
|
city_report[cid] = ['python', 'generate-city-report.py', '--existing-file', f'{base}/addresses-existing.geojson', '--output-dir', base]
|
|
if pu:
|
|
diff_paths[cid] = ['python', 'diff-highways.py', f'{base}/osm-paths.geojson', f'{base}/county-paths.geojson', '--output', f'{base}/diff-paths.geojson']
|
|
|
|
if dl_roads: sm['download-county-roads'] = dl_roads
|
|
if dl_addrs: sm['download-county-addresses'] = dl_addrs
|
|
if dl_paths: sm['download-county-paths'] = dl_paths
|
|
sm['download-osm-roads'] = osm_roads
|
|
sm['download-osm-addresses'] = osm_addrs
|
|
if osm_paths: sm['download-osm-paths'] = osm_paths
|
|
if conv_paths: sm['convert-paths'] = conv_paths
|
|
sm['diff-roads'] = diff_roads
|
|
sm['diff-addresses'] = diff_addrs
|
|
if diff_paths: sm['diff-paths'] = diff_paths
|
|
sm['generate-city-report'] = city_report
|
|
|
|
# Point-in-polygon check against the traced/imported boundary at
|
|
# /data/map-checker/boundary.geojson (see /map-checker and map-checker/README.md).
|
|
# Not per-county like the scripts above - one boundary can span multiple
|
|
# counties, so this checks all configured counties in a single run.
|
|
county_args = ' '.join(f'--county {shlex.quote(c["id"])}' for c in counties)
|
|
if county_args:
|
|
sm['boundary-check'] = (
|
|
f'python map-checker/check-boundary.py {county_args} '
|
|
f'--boundary /data/map-checker/boundary.geojson '
|
|
f'--output /data/map-checker/flagged.geojson'
|
|
)
|
|
|
|
return sm
|
|
|
|
@app.route('/api/run-script', methods=['POST'])
|
|
@require_auth
|
|
def run_script():
|
|
"""Execute a script in the background"""
|
|
data = request.json
|
|
script_name = data.get('script')
|
|
county = data.get('county', '')
|
|
|
|
if not script_name:
|
|
return jsonify({'error': 'No script specified'}), 400
|
|
|
|
script_map = get_script_map()
|
|
|
|
if script_name not in script_map:
|
|
return jsonify({'error': 'Unknown script'}), 400
|
|
|
|
script_config = script_map[script_name]
|
|
|
|
if isinstance(script_config, str):
|
|
cmd = ['bash', '-c', script_config]
|
|
elif isinstance(script_config, dict):
|
|
if not county:
|
|
return jsonify({'error': 'County required for this script'}), 400
|
|
if county not in script_config:
|
|
return jsonify({'error': f'County {county} not supported for {script_name}'}), 400
|
|
|
|
cmd_config = script_config[county]
|
|
if isinstance(cmd_config, str):
|
|
cmd = ['bash', '-c', cmd_config]
|
|
else:
|
|
cmd = list(cmd_config)
|
|
|
|
else:
|
|
return jsonify({'error': 'Invalid script configuration'}), 400
|
|
|
|
job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
|
|
county_name = next((c['name'] for c in get_counties() if c['id'] == county), county)
|
|
job = {
|
|
'id': job_id,
|
|
'script': script_name,
|
|
'label': str(data.get('label') or script_name)[:100],
|
|
'county': county,
|
|
'county_name': county_name,
|
|
'user': session.get('username'),
|
|
'started': _utcnow(),
|
|
'ended': None,
|
|
'status': 'running',
|
|
'exit_code': None,
|
|
}
|
|
with jobs_lock:
|
|
jobs[job_id] = job
|
|
process_logs[job_id] = []
|
|
_save_job(job)
|
|
_prune_jobs()
|
|
|
|
def run_command():
|
|
log_file = open(_job_log_path(job_id), 'w')
|
|
|
|
def log(line):
|
|
process_logs[job_id].append(line)
|
|
log_file.write(line)
|
|
log_file.flush()
|
|
|
|
try:
|
|
process = subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
cwd=os.path.dirname(os.path.dirname(__file__))
|
|
)
|
|
running_processes[job_id] = process
|
|
|
|
for line in process.stdout:
|
|
log(line)
|
|
|
|
process.wait()
|
|
job['exit_code'] = process.returncode
|
|
if job.get('cancelled'):
|
|
job['status'] = 'cancelled'
|
|
log(f"\n[Job cancelled by {job['cancelled']}]")
|
|
else:
|
|
job['status'] = 'success' if process.returncode == 0 else 'failed'
|
|
log(f"\n[Process completed with exit code {process.returncode}]")
|
|
|
|
except Exception as e:
|
|
job['status'] = 'failed'
|
|
log(f"\n[ERROR: {str(e)}]")
|
|
finally:
|
|
running_processes.pop(job_id, None)
|
|
log_file.close()
|
|
with jobs_lock:
|
|
job['ended'] = _utcnow()
|
|
_save_job(job)
|
|
# Finished logs live on disk; don't hold them in memory forever.
|
|
process_logs.pop(job_id, None)
|
|
_notify_job_finished(job)
|
|
|
|
thread = threading.Thread(target=run_command)
|
|
thread.daemon = True
|
|
thread.start()
|
|
|
|
return jsonify({'job_id': job_id, 'status': 'started', 'job': job})
|
|
|
|
@app.route('/api/jobs')
|
|
@require_auth
|
|
def list_jobs():
|
|
"""Most recent job overall, plus the most recent job for each script/county
|
|
pair, so every device can show the same banner and per-step status."""
|
|
with jobs_lock:
|
|
ordered = sorted(jobs.values(), key=lambda j: j['started'])
|
|
latest_by_step = {}
|
|
for job in ordered:
|
|
latest_by_step[f"{job['script']}|{job['county']}"] = job
|
|
return jsonify({
|
|
'latest': ordered[-1] if ordered else None,
|
|
'steps': latest_by_step,
|
|
'running': [j['id'] for j in ordered if j['status'] == 'running'],
|
|
})
|
|
|
|
@app.route('/api/job-status/<job_id>')
|
|
@require_auth
|
|
def job_status(job_id):
|
|
"""Get status and logs for a job"""
|
|
job = jobs.get(job_id)
|
|
if not job:
|
|
return jsonify({'error': 'Job not found'}), 404
|
|
return jsonify({'job_id': job_id, 'running': job['status'] == 'running',
|
|
'job': job, 'logs': _job_logs(job_id)})
|
|
|
|
@app.route('/api/cancel-job', methods=['POST'])
|
|
@require_auth
|
|
def cancel_job():
|
|
"""Cancel a running job"""
|
|
data = request.json
|
|
job_id = data.get('job_id')
|
|
|
|
if not job_id:
|
|
return jsonify({'error': 'No job ID specified'}), 400
|
|
if job_id not in running_processes:
|
|
return jsonify({'error': 'Job not found or already completed'}), 404
|
|
|
|
try:
|
|
jobs[job_id]['cancelled'] = session.get('username') or 'user'
|
|
running_processes[job_id].terminate()
|
|
return jsonify({'success': True})
|
|
except Exception as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
@app.route('/sw.js')
|
|
def service_worker():
|
|
"""Served from the site root so the worker's scope covers every page."""
|
|
resp = make_response(send_from_directory(app.static_folder, 'sw.js'))
|
|
resp.headers['Content-Type'] = 'application/javascript'
|
|
resp.headers['Cache-Control'] = 'no-cache'
|
|
return resp
|
|
|
|
@app.route('/manifest.webmanifest')
|
|
def web_manifest():
|
|
"""Lets phones install the dashboard; iOS only allows web push for installed apps."""
|
|
resp = make_response(send_from_directory(app.static_folder, 'manifest.webmanifest'))
|
|
resp.headers['Content-Type'] = 'application/manifest+json'
|
|
return resp
|
|
|
|
@app.route('/api/push/key')
|
|
@require_auth
|
|
def push_key():
|
|
if not _has_webpush:
|
|
return jsonify({'error': 'pywebpush not installed on the server'}), 503
|
|
return jsonify({'publicKey': _vapid_public_key()})
|
|
|
|
@app.route('/api/push/subscribe', methods=['POST'])
|
|
@require_auth
|
|
def push_subscribe():
|
|
sub = (request.json or {}).get('subscription') or {}
|
|
keys = sub.get('keys') or {}
|
|
if not (isinstance(sub.get('endpoint'), str) and sub['endpoint'].startswith('https://')
|
|
and keys.get('p256dh') and keys.get('auth')):
|
|
return jsonify({'error': 'Invalid subscription'}), 400
|
|
with push_lock:
|
|
subs = _load_push_subscriptions()
|
|
subs[sub['endpoint']] = {'subscription': {'endpoint': sub['endpoint'], 'keys': {
|
|
'p256dh': keys['p256dh'], 'auth': keys['auth']}},
|
|
'user': session.get('username'), 'created': _utcnow()}
|
|
_save_push_subscriptions(subs)
|
|
return jsonify({'success': True})
|
|
|
|
@app.route('/api/push/unsubscribe', methods=['POST'])
|
|
@require_auth
|
|
def push_unsubscribe():
|
|
endpoint = (request.json or {}).get('endpoint')
|
|
with push_lock:
|
|
subs = _load_push_subscriptions()
|
|
if subs.pop(endpoint, None) is not None:
|
|
_save_push_subscriptions(subs)
|
|
return jsonify({'success': True})
|
|
|
|
@app.route('/api/upload-file', methods=['POST'])
|
|
@require_auth
|
|
def upload_file():
|
|
"""Upload a shapefile ZIP to a path under /data"""
|
|
dest = request.form.get('dest')
|
|
if not dest or not dest.startswith('/data/'):
|
|
return jsonify({'error': 'Invalid destination path'}), 400
|
|
|
|
if 'file' not in request.files:
|
|
return jsonify({'error': 'No file provided'}), 400
|
|
|
|
f = request.files['file']
|
|
if not f.filename.lower().endswith('.zip'):
|
|
return jsonify({'error': 'File must be a ZIP archive (.zip)'}), 400
|
|
|
|
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
f.save(dest)
|
|
return jsonify({'success': True, 'path': dest})
|
|
|
|
@app.route('/api/clear-data', methods=['POST'])
|
|
@require_auth
|
|
def clear_data():
|
|
"""Delete all files from every county data directory under /data."""
|
|
counties = get_counties()
|
|
removed = []
|
|
errors = []
|
|
for county in counties:
|
|
county_dir = f'/data/{county["id"]}'
|
|
if not os.path.isdir(county_dir):
|
|
continue
|
|
for fname in os.listdir(county_dir):
|
|
fpath = os.path.join(county_dir, fname)
|
|
if os.path.isfile(fpath):
|
|
try:
|
|
os.remove(fpath)
|
|
removed.append(f'{county["id"]}/{fname}')
|
|
gz_copy = os.path.join(GZIP_CACHE_DIR, county['id'], fname + '.gz')
|
|
if os.path.exists(gz_copy):
|
|
os.remove(gz_copy)
|
|
except Exception as e:
|
|
errors.append(str(e))
|
|
if errors:
|
|
return jsonify({'error': '; '.join(errors)}), 500
|
|
return jsonify({'success': True, 'removed': removed})
|
|
|
|
@app.route('/api/system-stats')
|
|
def system_stats():
|
|
"""Return current CPU, memory, and disk usage percentages."""
|
|
if not _has_psutil:
|
|
return jsonify({'error': 'psutil not installed'}), 503
|
|
cpu = psutil.cpu_percent(interval=None)
|
|
mem = psutil.virtual_memory().percent
|
|
try:
|
|
disk = psutil.disk_usage('/data').percent
|
|
except Exception:
|
|
disk = psutil.disk_usage('/').percent
|
|
return jsonify({'cpu': cpu, 'mem': mem, 'disk': disk})
|
|
|
|
@app.route('/api/list-files')
|
|
def list_files():
|
|
"""List available GeoJSON files recursively with modification times."""
|
|
data_dir = '/data'
|
|
files = {'diff': [], 'osm': [], 'county': []}
|
|
|
|
if os.path.exists(data_dir):
|
|
for root, dirs, filenames in os.walk(data_dir):
|
|
dirs[:] = [d for d in dirs if d != 'osm_cache']
|
|
for filename in filenames:
|
|
if filename.endswith('.geojson'):
|
|
filepath = os.path.join(root, filename)
|
|
rel_path = os.path.relpath(filepath, data_dir).replace('\\', '/')
|
|
mtime = datetime.fromtimestamp(os.path.getmtime(filepath)).strftime('%Y-%m-%d %H:%M')
|
|
entry = {'path': rel_path, 'mtime': mtime}
|
|
if 'diff' in filename.lower():
|
|
files['diff'].append(entry)
|
|
elif 'osm' in filename.lower():
|
|
files['osm'].append(entry)
|
|
else:
|
|
files['county'].append(entry)
|
|
|
|
return jsonify(files)
|
|
|
|
GZIP_CACHE_DIR = '/data/.gzcache'
|
|
_gzip_lock = threading.Lock()
|
|
|
|
|
|
def _gzipped_copy(path, filename):
|
|
"""Path of a gzipped copy of path, (re)built when the source is newer.
|
|
GeoJSON compresses ~10x, which matters for 50-150 MB county files."""
|
|
gz_path = os.path.join(GZIP_CACHE_DIR, filename + '.gz')
|
|
with _gzip_lock:
|
|
if not os.path.exists(gz_path) or os.path.getmtime(gz_path) < os.path.getmtime(path):
|
|
os.makedirs(os.path.dirname(gz_path), exist_ok=True)
|
|
tmp = gz_path + '.tmp'
|
|
with open(path, 'rb') as src, gzip.open(tmp, 'wb', compresslevel=5) as dst:
|
|
shutil.copyfileobj(src, dst, 1024 * 1024)
|
|
os.replace(tmp, gz_path)
|
|
return gz_path
|
|
|
|
|
|
@app.route('/data/<path:filename>')
|
|
def serve_data(filename):
|
|
"""Serve files from /data (GeoJSON gzipped when the browser accepts it)"""
|
|
path = safe_join('/data', filename)
|
|
if (path and filename.endswith('.geojson') and os.path.isfile(path)
|
|
and 'gzip' in request.headers.get('Accept-Encoding', '')):
|
|
resp = make_response(send_file(_gzipped_copy(path, filename), mimetype='application/geo+json'))
|
|
resp.headers['Content-Encoding'] = 'gzip'
|
|
resp.headers['Vary'] = 'Accept-Encoding'
|
|
return resp
|
|
return send_from_directory('/data', filename)
|
|
|
|
# ── Config file download / upload ──────────────────────────────────────────────
|
|
|
|
_ALLOWED_CONFIGS = {'counties.yml', 'exceptions.yml', 'exclusions.yml'}
|
|
|
|
@app.route('/api/config/<filename>', methods=['GET'])
|
|
@require_auth
|
|
def download_config(filename):
|
|
if filename not in _ALLOWED_CONFIGS:
|
|
return jsonify({'error': 'Not found'}), 404
|
|
return send_from_directory('/data', filename, as_attachment=True)
|
|
|
|
@app.route('/api/config/<filename>', methods=['POST'])
|
|
@require_auth
|
|
def upload_config(filename):
|
|
if filename not in _ALLOWED_CONFIGS:
|
|
return jsonify({'error': 'Not found'}), 404
|
|
if 'file' not in request.files:
|
|
return jsonify({'error': 'No file provided'}), 400
|
|
f = request.files['file']
|
|
try:
|
|
content = f.read().decode('utf-8')
|
|
data = yaml.safe_load(content)
|
|
if filename == 'counties.yml' and not isinstance((data or {}).get('counties'), list):
|
|
return jsonify({'error': 'Invalid counties.yml: missing counties list'}), 400
|
|
if filename == 'exceptions.yml' and not isinstance((data or {}).get('corrections'), list):
|
|
return jsonify({'error': 'Invalid exceptions.yml: missing corrections list'}), 400
|
|
if filename == 'exclusions.yml' and not isinstance((data or {}).get('exclusions'), list):
|
|
return jsonify({'error': 'Invalid exclusions.yml: missing exclusions list'}), 400
|
|
except Exception as e:
|
|
return jsonify({'error': f'Invalid YAML: {e}'}), 400
|
|
dest = f'/data/{filename}'
|
|
tmp = dest + '.tmp'
|
|
with open(tmp, 'w') as out:
|
|
out.write(content)
|
|
os.replace(tmp, dest)
|
|
return jsonify({'success': True})
|
|
|
|
# ── Exclusions config ─────────────────────────────────────────────────────────
|
|
|
|
EXCLUSIONS_FILE = '/data/exclusions.yml'
|
|
|
|
|
|
def _ensure_exclusions_file():
|
|
if not os.path.exists(EXCLUSIONS_FILE):
|
|
os.makedirs(os.path.dirname(EXCLUSIONS_FILE), exist_ok=True)
|
|
with open(EXCLUSIONS_FILE, 'w') as f:
|
|
yaml.dump({'exclusions': []}, f, default_flow_style=False, allow_unicode=True)
|
|
|
|
_ensure_exclusions_file()
|
|
|
|
|
|
def get_exclusions():
|
|
try:
|
|
with open(EXCLUSIONS_FILE) as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return data.get('exclusions', [])
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
@app.route('/exclusions')
|
|
def exclusions_page():
|
|
return render_template('exclusions.html', current_user=session.get('username'))
|
|
|
|
|
|
@app.route('/api/exclusions', methods=['GET'])
|
|
def get_exclusions_api():
|
|
return jsonify({'exclusions': get_exclusions()})
|
|
|
|
|
|
@app.route('/api/exclusions', methods=['POST'])
|
|
@require_auth
|
|
def save_exclusions():
|
|
data = request.json
|
|
items = data.get('exclusions', [])
|
|
for item in items:
|
|
if not isinstance(item.get('field'), str) or not item['field'].strip():
|
|
return jsonify({'error': 'Each exclusion must have a non-empty "field"'}), 400
|
|
if not isinstance(item.get('value'), str) or not item['value'].strip():
|
|
return jsonify({'error': 'Each exclusion must have a non-empty "value"'}), 400
|
|
tmp = EXCLUSIONS_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
yaml.dump({'exclusions': items}, f, default_flow_style=False, allow_unicode=True)
|
|
os.replace(tmp, EXCLUSIONS_FILE)
|
|
return jsonify({'success': True})
|
|
|
|
|
|
@app.route('/api/exclusions/add', methods=['POST'])
|
|
@require_auth
|
|
def add_exclusions():
|
|
"""Append new exclusion entries, skipping duplicates."""
|
|
data = request.json
|
|
new_items = data.get('exclusions', [])
|
|
existing = get_exclusions()
|
|
existing_keys = {(e['field'], e['value']) for e in existing}
|
|
added = 0
|
|
for item in new_items:
|
|
field = (item.get('field') or '').strip()
|
|
value = (item.get('value') or '').strip()
|
|
if not field or not value:
|
|
continue
|
|
if (field, value) not in existing_keys:
|
|
existing.append({'field': field, 'value': value})
|
|
existing_keys.add((field, value))
|
|
added += 1
|
|
tmp = EXCLUSIONS_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
yaml.dump({'exclusions': existing}, f, default_flow_style=False, allow_unicode=True)
|
|
os.replace(tmp, EXCLUSIONS_FILE)
|
|
return jsonify({'success': True, 'added': added})
|
|
|
|
# ── Counties config ────────────────────────────────────────────────────────────
|
|
|
|
COUNTIES_FILE = '/data/counties.yml'
|
|
|
|
DEFAULT_COUNTIES = [
|
|
{
|
|
'id': 'sumter',
|
|
'name': 'Sumter County',
|
|
'state': 'Florida',
|
|
'color': '#28a745',
|
|
'roads_url': 'https://www.arcgis.com/sharing/rest/content/items/9177e17c72d3433aa79630c7eda84add/data',
|
|
'addresses_url': 'https://www.arcgis.com/sharing/rest/content/items/c75c5aac13a648968c5596b0665be28b/data',
|
|
'paths_url': 'upload',
|
|
'pinned_cities': ['The Villages'],
|
|
},
|
|
{
|
|
'id': 'lake',
|
|
'name': 'Lake County',
|
|
'state': 'Florida',
|
|
'color': '#17a2b8',
|
|
'roads_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Streets.zip',
|
|
'addresses_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Addresspoints.zip',
|
|
'paths_url': '',
|
|
'pinned_cities': ['The Villages'],
|
|
},
|
|
]
|
|
|
|
def _ensure_counties_file():
|
|
if not os.path.exists(COUNTIES_FILE):
|
|
os.makedirs(os.path.dirname(COUNTIES_FILE), exist_ok=True)
|
|
with open(COUNTIES_FILE, 'w') as f:
|
|
yaml.dump({'counties': DEFAULT_COUNTIES}, f,
|
|
default_flow_style=False, allow_unicode=True)
|
|
|
|
_ensure_counties_file()
|
|
|
|
def get_counties():
|
|
try:
|
|
with open(COUNTIES_FILE) as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return data.get('counties', DEFAULT_COUNTIES)
|
|
except Exception:
|
|
return DEFAULT_COUNTIES
|
|
|
|
@app.route('/counties')
|
|
def counties_page():
|
|
return render_template('counties.html', current_user=session.get('username'))
|
|
|
|
@app.route('/api/counties', methods=['GET'])
|
|
def get_counties_api():
|
|
try:
|
|
return jsonify({'counties': get_counties()})
|
|
except Exception as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
@app.route('/api/counties', methods=['POST'])
|
|
@require_auth
|
|
def save_counties():
|
|
data = request.json
|
|
counties = data.get('counties', [])
|
|
for c in counties:
|
|
if not c.get('id', '').strip():
|
|
return jsonify({'error': 'Each county must have a non-empty id'}), 400
|
|
if not c.get('name', '').strip():
|
|
return jsonify({'error': 'Each county must have a non-empty name'}), 400
|
|
tmp = COUNTIES_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
yaml.dump({'counties': counties}, f, default_flow_style=False, allow_unicode=True)
|
|
os.replace(tmp, COUNTIES_FILE)
|
|
return jsonify({'success': True})
|
|
|
|
# ── Exceptions config ──────────────────────────────────────────────────────────
|
|
|
|
EXCEPTIONS_FILE = '/data/exceptions.yml'
|
|
|
|
DEFAULT_EXCEPTIONS = [
|
|
{'from': "D Angelo Lane", 'to': "D'Angelo Lane"},
|
|
{'from': "Lajolla Circle", 'to': "La Jolla Circle"},
|
|
{'from': "Gardena Court", 'to': "Gardenia Court"},
|
|
{'from': "Glenmont Court", 'to': "Glenmount Court"},
|
|
{'from': "Pawleys Island Path", 'to': "Pawley's Island Path"},
|
|
{'from': "Oday Street", 'to': "O'Day Street"},
|
|
{'from': "Obrien Place", 'to': "O'Brien Place"},
|
|
{'from': "Ohara Court", 'to': "O'Hara Court"},
|
|
]
|
|
|
|
def _ensure_exceptions_file():
|
|
if not os.path.exists(EXCEPTIONS_FILE):
|
|
os.makedirs(os.path.dirname(EXCEPTIONS_FILE), exist_ok=True)
|
|
with open(EXCEPTIONS_FILE, 'w') as f:
|
|
yaml.dump({'corrections': DEFAULT_EXCEPTIONS}, f,
|
|
default_flow_style=False, allow_unicode=True)
|
|
|
|
_ensure_exceptions_file()
|
|
|
|
|
|
@app.route('/exceptions')
|
|
def exceptions_page():
|
|
return render_template('exceptions.html', current_user=session.get('username'))
|
|
|
|
|
|
@app.route('/api/exceptions', methods=['GET'])
|
|
def get_exceptions():
|
|
try:
|
|
with open(EXCEPTIONS_FILE) as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return jsonify({'corrections': data.get('corrections', [])})
|
|
except Exception as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@app.route('/api/exceptions', methods=['POST'])
|
|
@require_auth
|
|
def save_exceptions():
|
|
data = request.json
|
|
corrections = data.get('corrections', [])
|
|
for item in corrections:
|
|
if not isinstance(item.get('from'), str) or not item['from'].strip():
|
|
return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400
|
|
if not isinstance(item.get('to'), str) or not item['to'].strip():
|
|
return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400
|
|
tmp = EXCEPTIONS_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
yaml.dump({'corrections': corrections}, f,
|
|
default_flow_style=False, allow_unicode=True)
|
|
os.replace(tmp, EXCEPTIONS_FILE)
|
|
return jsonify({'success': True})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
app.run(host='0.0.0.0', port=5000, debug=True)
|