#!/usr/bin/env python3
"""
Flask web server for The Villages Import Tools
"""
from functools import wraps
from flask import Flask, render_template, jsonify, request, send_from_directory, send_file, session, redirect, url_for, make_response
from werkzeug.utils import safe_join
from werkzeug.security import generate_password_hash, check_password_hash
import subprocess
import os
import re
import secrets
import shlex
import shutil
import threading
import json
import base64
import gzip
import logging
from datetime import datetime, timezone
import yaml
try:
    import psutil
    _has_psutil = True
except ImportError:
    _has_psutil = False
try:
    from pywebpush import webpush, WebPushException
    from py_vapid import Vapid
    from cryptography.hazmat.primitives import serialization
    _has_webpush = True
except ImportError:
    _has_webpush = False

app = Flask(__name__, static_folder='static', template_folder='templates')

# ── Secret key ────────────────────────────────────────────────────────────────

def _get_or_create_secret_key():
    key = os.environ.get('FLASK_SECRET_KEY', '')
    if key:
        return key
    key_file = '/data/.secret_key'
    if os.path.exists(key_file):
        with open(key_file) as f:
            return f.read().strip()
    key = secrets.token_hex(32)
    try:
        os.makedirs('/data', exist_ok=True)
        with open(key_file, 'w') as f:
            f.write(key)
    except Exception:
        pass
    return key

app.secret_key = _get_or_create_secret_key()

# ── Users / auth ───────────────────────────────────────────────────────────────

USERS_FILE = '/data/users.yml'


def _load_users():
    try:
        with open(USERS_FILE) as f:
            data = yaml.safe_load(f) or {}
        return {u['username']: u['password_hash'] for u in data.get('users', [])}
    except Exception:
        return {}


def _save_users(users_dict):
    users_list = [{'username': u, 'password_hash': h} for u, h in sorted(users_dict.items())]
    tmp = USERS_FILE + '.tmp'
    with open(tmp, 'w') as f:
        yaml.dump({'users': users_list}, f, default_flow_style=False, allow_unicode=True)
    os.replace(tmp, USERS_FILE)


def _seed_admin():
    """Create or update the admin user from the ADMIN_PASSWORD env var."""
    password = os.environ.get('ADMIN_PASSWORD', '')
    if not password:
        return
    users = _load_users()
    users['admin'] = generate_password_hash(password)
    _save_users(users)


_seed_admin()


def require_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        if not session.get('username'):
            if request.is_json or request.path.startswith('/api/'):
                return jsonify({'error': 'Authentication required'}), 401
            return redirect(url_for('login_page', next=request.path))
        return f(*args, **kwargs)
    return decorated


@app.route('/login', methods=['GET'])
def login_page():
    if session.get('username'):
        return redirect(request.args.get('next', '/'))
    return render_template('login.html', next=request.args.get('next', '/'), error=None)


@app.route('/login', methods=['POST'])
def login():
    username = request.form.get('username', '').strip()
    password = request.form.get('password', '')
    next_url = request.form.get('next', '/')
    users = _load_users()
    if username and username in users and check_password_hash(users[username], password):
        session['username'] = username
        return redirect(next_url)
    return render_template('login.html', next=next_url, error='Invalid username or password')


@app.route('/logout', methods=['POST'])
def logout():
    session.clear()
    return redirect('/')


@app.route('/api/auth-status')
def auth_status():
    username = session.get('username')
    return jsonify({'authenticated': bool(username), 'username': username or ''})

# ── Job registry ──────────────────────────────────────────────────────────────
# Every run is recorded under /data/jobs (metadata JSON + log file) so any
# logged-in user, on any device, can see what ran, when, and how it ended -
# even after the job finished or the server restarted.

JOBS_DIR = '/data/jobs'
JOBS_KEEP = 200

running_processes = {}
jobs = {}           # job_id -> metadata dict (see _new_job)
process_logs = {}   # job_id -> list of log lines
jobs_lock = threading.Lock()


def _utcnow():
    return datetime.now(timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z')


def _job_meta_path(job_id):
    return os.path.join(JOBS_DIR, f'{job_id}.json')


def _job_log_path(job_id):
    return os.path.join(JOBS_DIR, f'{job_id}.log')


def _save_job(job):
    os.makedirs(JOBS_DIR, exist_ok=True)
    tmp = _job_meta_path(job['id']) + '.tmp'
    with open(tmp, 'w') as f:
        json.dump(job, f)
    os.replace(tmp, _job_meta_path(job['id']))


def _load_jobs():
    """Load job history from disk. Jobs left 'running' by a previous server
    process can no longer be watched, so they are marked interrupted."""
    if not os.path.isdir(JOBS_DIR):
        return
    for fname in os.listdir(JOBS_DIR):
        if not fname.endswith('.json'):
            continue
        try:
            with open(os.path.join(JOBS_DIR, fname)) as f:
                job = json.load(f)
        except (OSError, ValueError):
            continue
        if job.get('status') == 'running':
            job['status'] = 'interrupted'
            job['ended'] = job.get('ended') or _utcnow()
            _save_job(job)
        jobs[job['id']] = job


def _prune_jobs():
    """Keep only the newest JOBS_KEEP jobs on disk and in memory."""
    finished = sorted((j for j in jobs.values() if j['status'] != 'running'),
                      key=lambda j: j['started'])
    for job in finished[:max(0, len(jobs) - JOBS_KEEP)]:
        jobs.pop(job['id'], None)
        process_logs.pop(job['id'], None)
        for path in (_job_meta_path(job['id']), _job_log_path(job['id'])):
            try:
                os.remove(path)
            except OSError:
                pass


def _job_logs(job_id):
    if job_id in process_logs:
        return process_logs[job_id]
    try:
        with open(_job_log_path(job_id)) as f:
            return f.readlines()
    except OSError:
        return []


_load_jobs()


# ── Web push: notify subscribed browsers when a job finishes ─────────────────
# VAPID keys are generated once into /data (like the Flask secret key), or set
# VAPID_PRIVATE_KEY_FILE to use your own PEM. Browsers only allow push on HTTPS
# (or http://localhost).

VAPID_KEY_FILE = os.environ.get('VAPID_PRIVATE_KEY_FILE', '/data/.vapid_private.pem')
VAPID_SUBJECT = os.environ.get('VAPID_SUBJECT', 'mailto:admin@example.org')
PUSH_SUBSCRIPTIONS_FILE = '/data/push-subscriptions.json'
push_lock = threading.Lock()
_vapid = None


def _get_vapid():
    global _vapid
    if _vapid is None and _has_webpush:
        if os.path.exists(VAPID_KEY_FILE):
            _vapid = Vapid.from_file(VAPID_KEY_FILE)
        else:
            _vapid = Vapid()
            _vapid.generate_keys()
            _vapid.save_key(VAPID_KEY_FILE)
            os.chmod(VAPID_KEY_FILE, 0o600)
    return _vapid


def _vapid_public_key():
    raw = _get_vapid().public_key.public_bytes(
        serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint)
    return base64.urlsafe_b64encode(raw).decode().rstrip('=')


def _load_push_subscriptions():
    try:
        with open(PUSH_SUBSCRIPTIONS_FILE) as f:
            return json.load(f)
    except (OSError, ValueError):
        return {}


def _save_push_subscriptions(subs):
    tmp = PUSH_SUBSCRIPTIONS_FILE + '.tmp'
    with open(tmp, 'w') as f:
        json.dump(subs, f)
    os.replace(tmp, PUSH_SUBSCRIPTIONS_FILE)


def _notify_job_finished(job):
    """Push a notification about a finished job to every subscribed browser.
    Subscriptions the push service reports as gone are dropped."""
    if not _has_webpush:
        return
    with push_lock:
        subs = _load_push_subscriptions()
    if not subs:
        return
    payload = json.dumps({'job': job})
    gone = []
    for endpoint, entry in subs.items():
        try:
            webpush(entry['subscription'], payload,
                    vapid_private_key=VAPID_KEY_FILE,
                    vapid_claims={'sub': VAPID_SUBJECT},
                    ttl=24 * 3600)
        except WebPushException as e:
            if e.response is not None and e.response.status_code in (404, 410):
                gone.append(endpoint)
            else:
                logging.warning('Web push to %s failed: %s', endpoint[:60], e)
        except Exception as e:
            logging.warning('Web push to %s failed: %s', endpoint[:60], e)
    if gone:
        with push_lock:
            subs = _load_push_subscriptions()
            for endpoint in gone:
                subs.pop(endpoint, None)
            _save_push_subscriptions(subs)


def _cleanup_legacy_structure():
    """Remove dated folders and symlink from the old /data/YYMMDD layout,
    migrating any files up to /data/{county}/ before deleting."""
    data_dir = '/data'
    if not os.path.exists(data_dir):
        return

    symlink = os.path.join(data_dir, 'latest')
    if os.path.islink(symlink):
        os.unlink(symlink)

    dated_pattern = re.compile(r'^\d{6}$')
    for entry in os.listdir(data_dir):
        entry_path = os.path.join(data_dir, entry)
        if os.path.isdir(entry_path) and dated_pattern.match(entry):
            for county_dir in os.listdir(entry_path):
                src_county = os.path.join(entry_path, county_dir)
                if os.path.isdir(src_county):
                    dst_county = os.path.join(data_dir, county_dir)
                    os.makedirs(dst_county, exist_ok=True)
                    for fname in os.listdir(src_county):
                        src_file = os.path.join(src_county, fname)
                        dst_file = os.path.join(dst_county, fname)
                        if os.path.isfile(src_file) and not os.path.exists(dst_file):
                            shutil.move(src_file, dst_file)
            shutil.rmtree(entry_path)

_cleanup_legacy_structure()


def _list_data_files(counties):
    """Files in each county's data folder, newest first: {county_id: [{name, ts}]}."""
    data_files = {}
    for county in counties:
        county_dir = f'/data/{county["id"]}'
        if os.path.isdir(county_dir):
            files = []
            for fname in sorted(os.listdir(county_dir)):
                fpath = os.path.join(county_dir, fname)
                if os.path.isfile(fpath):
                    files.append({'name': fname, 'ts': int(os.path.getmtime(fpath))})
            if files:
                files.sort(key=lambda f: f['ts'], reverse=True)
                data_files[county['id']] = files
    return data_files


@app.route('/api/data-files')
def data_files_api():
    return jsonify(_list_data_files(get_counties()))


@app.route('/')
def index():
    """Main index page with script execution buttons"""
    counties = get_counties()
    data_files = _list_data_files(counties)

    import json as _json
    return render_template('index.html',
                           counties=counties,
                           counties_json=_json.dumps(counties),
                           data_files_json=_json.dumps(data_files),
                           current_user=session.get('username'))

@app.route('/map')
def map_viewer():
    """Map viewer page"""
    return render_template('map.html')

# ── Reports ────────────────────────────────────────────────────────────────

@app.route('/reports')
def reports_page():
    """Reports index page"""
    return render_template('reports.html', current_user=session.get('username'))

@app.route('/reports/city-boundaries')
def city_boundaries_report():
    """City name agreement report: county vs OSM addr:city, on a map"""
    counties = get_counties()
    available = []
    pinned_cities = {}
    for county in counties:
        path = f'/data/{county["id"]}/city-comparison.geojson'
        if os.path.isfile(path):
            available.append(county['id'])
        pinned_cities[county['id']] = county.get('pinned_cities') or []
    return render_template('city-boundaries-report.html',
                           counties=counties,
                           available_counties=available,
                           pinned_cities=pinned_cities,
                           current_user=session.get('username'))

# ── map-checker: hand-traced boundary tool ──────────────────────────────────
#
# thevillages.com's Terms of Use prohibit automated access, so this only ever
# saves what a human draws by hand while looking at the official map
# themselves - nothing here fetches from thevillages.com. See map-checker/README.md.

MAP_CHECKER_DIR = '/data/map-checker'
MAP_CHECKER_BOUNDARY_FILE = os.path.join(MAP_CHECKER_DIR, 'boundary.geojson')
MAP_CHECKER_DRAFT_FILE = os.path.join(MAP_CHECKER_DIR, 'draft.json')

@app.route('/map-checker')
@require_auth
def map_checker_page():
    """Boundary tracer: hand-trace a polygon against a reference map, save as GeoJSON.

    Requires login so tracing work is never done (and lost) as an anonymous session."""
    return render_template('boundary-tracer.html',
                           counties=get_counties(),
                           current_user=session.get('username'))

@app.route('/api/map-checker/boundary', methods=['GET'])
@require_auth
def get_map_checker_boundary():
    if not os.path.isfile(MAP_CHECKER_BOUNDARY_FILE):
        return jsonify({'type': 'FeatureCollection', 'features': []})
    with open(MAP_CHECKER_BOUNDARY_FILE) as f:
        return jsonify(json.load(f))

@app.route('/api/map-checker/boundary', methods=['POST'])
@require_auth
def save_map_checker_boundary():
    data = request.json or {}
    if data.get('type') != 'FeatureCollection' or not isinstance(data.get('features'), list):
        return jsonify({'error': 'Expected a GeoJSON FeatureCollection'}), 400
    for feat in data['features']:
        geom_type = ((feat.get('geometry') or {}).get('type'))
        if geom_type not in ('Polygon', 'MultiPolygon'):
            return jsonify({'error': f'Unsupported geometry type: {geom_type}'}), 400
    os.makedirs(MAP_CHECKER_DIR, exist_ok=True)
    tmp = MAP_CHECKER_BOUNDARY_FILE + '.tmp'
    with open(tmp, 'w') as f:
        json.dump(data, f)
    os.replace(tmp, MAP_CHECKER_BOUNDARY_FILE)
    return jsonify({'success': True, 'features': len(data['features'])})


def _is_latlng_list(val):
    if not isinstance(val, list):
        return False
    for pt in val:
        if not (isinstance(pt, list) and len(pt) == 2 and
                all(isinstance(c, (int, float)) for c in pt)):
            return False
    return True


@app.route('/api/map-checker/draft', methods=['GET'])
@require_auth
def get_map_checker_draft():
    if not os.path.isfile(MAP_CHECKER_DRAFT_FILE):
        return jsonify({'polygons': [], 'inProgress': []})
    with open(MAP_CHECKER_DRAFT_FILE) as f:
        return jsonify(json.load(f))

@app.route('/api/map-checker/draft', methods=['POST'])
@require_auth
def save_map_checker_draft():
    """Autosave endpoint hit after every vertex/edit so in-progress tracing
    survives a crashed tab or dropped connection, even before the user
    explicitly finishes a polygon or clicks Save to server."""
    data = request.json or {}
    polygons = data.get('polygons')
    in_progress = data.get('inProgress', [])
    if not isinstance(polygons, list):
        return jsonify({'error': 'Expected a polygons list'}), 400
    for p in polygons:
        if not (isinstance(p, dict) and isinstance(p.get('name'), str) and _is_latlng_list(p.get('latlngs'))):
            return jsonify({'error': 'Malformed polygon entry'}), 400
    if not _is_latlng_list(in_progress):
        return jsonify({'error': 'Malformed inProgress point list'}), 400
    os.makedirs(MAP_CHECKER_DIR, exist_ok=True)
    tmp = MAP_CHECKER_DRAFT_FILE + '.tmp'
    with open(tmp, 'w') as f:
        json.dump({'polygons': polygons, 'inProgress': in_progress}, f)
    os.replace(tmp, MAP_CHECKER_DRAFT_FILE)
    return jsonify({'success': True})

def get_script_map():
    """Build script map dynamically from county configuration."""
    counties = get_counties()

    sm = {
        'ls': 'ls -alR /data',
    }

    dl_roads = {}; dl_addrs = {}; dl_paths = {}
    osm_roads = {}; osm_addrs = {}; osm_paths = {}
    conv_paths = {}
    diff_roads = {}; diff_addrs = {}; diff_paths = {}
    city_report = {}

    for c in counties:
        cid   = c['id']
        name  = c['name']
        state = c['state']
        base  = f'/data/{cid}'

        # Each county download also converts the shapefile to GeoJSON, so the
        # diff can run straight after it. Uploaded paths are converted by a
        # separate convert-paths job that the dashboard starts after the upload.
        conv_roads = f'python shp-to-geojson.py {base}/roads.shp.zip {base}/county-roads.geojson'
        conv_addrs = f'python convert-addresses.py {base}/addresses.shp.zip {base}/county-addresses.geojson'
        conv_paths_cmd = f'python shp-to-geojson.py {base}/paths.shp.zip {base}/county-paths.geojson'

        if c.get('roads_url'):
            dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip && {conv_roads}'
        if c.get('addresses_url'):
            dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip && {conv_addrs}'
        pu = c.get('paths_url', '')
        if pu == 'upload':
            dl_paths[cid] = {'type': 'upload', 'dest': f'{base}/paths.shp.zip'}
            conv_paths[cid] = conv_paths_cmd
        elif pu:
            dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip && {conv_paths_cmd}'

        osm_roads[cid] = ['python', 'download-overpass.py', '--type', 'highways',  name, state, f'{base}/osm-roads.geojson']
        osm_addrs[cid] = ['python', 'download-overpass.py', '--type', 'addresses', name, state, f'{base}/osm-addresses.geojson']
        if pu:
            osm_paths[cid] = ['python', 'download-overpass.py', '--type', 'paths', name, state, f'{base}/osm-paths.geojson']

        diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson']
        diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base]
        city_report[cid] = ['python', 'generate-city-report.py', '--existing-file', f'{base}/addresses-existing.geojson', '--output-dir', base]
        if pu:
            diff_paths[cid] = ['python', 'diff-highways.py', f'{base}/osm-paths.geojson', f'{base}/county-paths.geojson', '--output', f'{base}/diff-paths.geojson']

    if dl_roads:   sm['download-county-roads']    = dl_roads
    if dl_addrs:   sm['download-county-addresses'] = dl_addrs
    if dl_paths:   sm['download-county-paths']     = dl_paths
    sm['download-osm-roads']    = osm_roads
    sm['download-osm-addresses'] = osm_addrs
    if osm_paths:  sm['download-osm-paths']  = osm_paths
    if conv_paths: sm['convert-paths'] = conv_paths
    sm['diff-roads']    = diff_roads
    sm['diff-addresses'] = diff_addrs
    if diff_paths: sm['diff-paths'] = diff_paths
    sm['generate-city-report'] = city_report

    # Point-in-polygon check against the traced/imported boundary at
    # /data/map-checker/boundary.geojson (see /map-checker and map-checker/README.md).
    # Not per-county like the scripts above - one boundary can span multiple
    # counties, so this checks all configured counties in a single run.
    county_args = ' '.join(f'--county {shlex.quote(c["id"])}' for c in counties)
    if county_args:
        sm['boundary-check'] = (
            f'python map-checker/check-boundary.py {county_args} '
            f'--boundary /data/map-checker/boundary.geojson '
            f'--output /data/map-checker/flagged.geojson'
        )

    return sm

@app.route('/api/run-script', methods=['POST'])
@require_auth
def run_script():
    """Execute a script in the background"""
    data = request.json
    script_name = data.get('script')
    county = data.get('county', '')

    if not script_name:
        return jsonify({'error': 'No script specified'}), 400

    script_map = get_script_map()

    if script_name not in script_map:
        return jsonify({'error': 'Unknown script'}), 400

    script_config = script_map[script_name]

    if isinstance(script_config, str):
        cmd = ['bash', '-c', script_config]
    elif isinstance(script_config, dict):
        if not county:
            return jsonify({'error': 'County required for this script'}), 400
        if county not in script_config:
            return jsonify({'error': f'County {county} not supported for {script_name}'}), 400

        cmd_config = script_config[county]
        if isinstance(cmd_config, str):
            cmd = ['bash', '-c', cmd_config]
        else:
            cmd = list(cmd_config)

    else:
        return jsonify({'error': 'Invalid script configuration'}), 400

    job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
    county_name = next((c['name'] for c in get_counties() if c['id'] == county), county)
    job = {
        'id': job_id,
        'script': script_name,
        'label': str(data.get('label') or script_name)[:100],
        'county': county,
        'county_name': county_name,
        'user': session.get('username'),
        'started': _utcnow(),
        'ended': None,
        'status': 'running',
        'exit_code': None,
    }
    with jobs_lock:
        jobs[job_id] = job
        process_logs[job_id] = []
        _save_job(job)
        _prune_jobs()

    def run_command():
        log_file = open(_job_log_path(job_id), 'w')

        def log(line):
            process_logs[job_id].append(line)
            log_file.write(line)
            log_file.flush()

        try:
            process = subprocess.Popen(
                cmd,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                text=True,
                cwd=os.path.dirname(os.path.dirname(__file__))
            )
            running_processes[job_id] = process

            for line in process.stdout:
                log(line)

            process.wait()
            job['exit_code'] = process.returncode
            if job.get('cancelled'):
                job['status'] = 'cancelled'
                log(f"\n[Job cancelled by {job['cancelled']}]")
            else:
                job['status'] = 'success' if process.returncode == 0 else 'failed'
                log(f"\n[Process completed with exit code {process.returncode}]")

        except Exception as e:
            job['status'] = 'failed'
            log(f"\n[ERROR: {str(e)}]")
        finally:
            running_processes.pop(job_id, None)
            log_file.close()
            with jobs_lock:
                job['ended'] = _utcnow()
                _save_job(job)
                # Finished logs live on disk; don't hold them in memory forever.
                process_logs.pop(job_id, None)
            _notify_job_finished(job)

    thread = threading.Thread(target=run_command)
    thread.daemon = True
    thread.start()

    return jsonify({'job_id': job_id, 'status': 'started', 'job': job})

@app.route('/api/jobs')
@require_auth
def list_jobs():
    """Most recent job overall, plus the most recent job for each script/county
    pair, so every device can show the same banner and per-step status."""
    with jobs_lock:
        ordered = sorted(jobs.values(), key=lambda j: j['started'])
    latest_by_step = {}
    for job in ordered:
        latest_by_step[f"{job['script']}|{job['county']}"] = job
    return jsonify({
        'latest': ordered[-1] if ordered else None,
        'steps': latest_by_step,
        'running': [j['id'] for j in ordered if j['status'] == 'running'],
    })

@app.route('/api/job-status/<job_id>')
@require_auth
def job_status(job_id):
    """Get status and logs for a job"""
    job = jobs.get(job_id)
    if not job:
        return jsonify({'error': 'Job not found'}), 404
    return jsonify({'job_id': job_id, 'running': job['status'] == 'running',
                    'job': job, 'logs': _job_logs(job_id)})

@app.route('/api/cancel-job', methods=['POST'])
@require_auth
def cancel_job():
    """Cancel a running job"""
    data = request.json
    job_id = data.get('job_id')

    if not job_id:
        return jsonify({'error': 'No job ID specified'}), 400
    if job_id not in running_processes:
        return jsonify({'error': 'Job not found or already completed'}), 404

    try:
        jobs[job_id]['cancelled'] = session.get('username') or 'user'
        running_processes[job_id].terminate()
        return jsonify({'success': True})
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/sw.js')
def service_worker():
    """Served from the site root so the worker's scope covers every page."""
    resp = make_response(send_from_directory(app.static_folder, 'sw.js'))
    resp.headers['Content-Type'] = 'application/javascript'
    resp.headers['Cache-Control'] = 'no-cache'
    return resp

@app.route('/manifest.webmanifest')
def web_manifest():
    """Lets phones install the dashboard; iOS only allows web push for installed apps."""
    resp = make_response(send_from_directory(app.static_folder, 'manifest.webmanifest'))
    resp.headers['Content-Type'] = 'application/manifest+json'
    return resp

@app.route('/api/push/key')
@require_auth
def push_key():
    if not _has_webpush:
        return jsonify({'error': 'pywebpush not installed on the server'}), 503
    return jsonify({'publicKey': _vapid_public_key()})

@app.route('/api/push/subscribe', methods=['POST'])
@require_auth
def push_subscribe():
    sub = (request.json or {}).get('subscription') or {}
    keys = sub.get('keys') or {}
    if not (isinstance(sub.get('endpoint'), str) and sub['endpoint'].startswith('https://')
            and keys.get('p256dh') and keys.get('auth')):
        return jsonify({'error': 'Invalid subscription'}), 400
    with push_lock:
        subs = _load_push_subscriptions()
        subs[sub['endpoint']] = {'subscription': {'endpoint': sub['endpoint'], 'keys': {
            'p256dh': keys['p256dh'], 'auth': keys['auth']}},
            'user': session.get('username'), 'created': _utcnow()}
        _save_push_subscriptions(subs)
    return jsonify({'success': True})

@app.route('/api/push/unsubscribe', methods=['POST'])
@require_auth
def push_unsubscribe():
    endpoint = (request.json or {}).get('endpoint')
    with push_lock:
        subs = _load_push_subscriptions()
        if subs.pop(endpoint, None) is not None:
            _save_push_subscriptions(subs)
    return jsonify({'success': True})

@app.route('/api/upload-file', methods=['POST'])
@require_auth
def upload_file():
    """Upload a shapefile ZIP to a path under /data"""
    dest = request.form.get('dest')
    if not dest or not dest.startswith('/data/'):
        return jsonify({'error': 'Invalid destination path'}), 400

    if 'file' not in request.files:
        return jsonify({'error': 'No file provided'}), 400

    f = request.files['file']
    if not f.filename.lower().endswith('.zip'):
        return jsonify({'error': 'File must be a ZIP archive (.zip)'}), 400

    os.makedirs(os.path.dirname(dest), exist_ok=True)
    f.save(dest)
    return jsonify({'success': True, 'path': dest})

@app.route('/api/clear-data', methods=['POST'])
@require_auth
def clear_data():
    """Delete all files from every county data directory under /data."""
    counties = get_counties()
    removed = []
    errors = []
    for county in counties:
        county_dir = f'/data/{county["id"]}'
        if not os.path.isdir(county_dir):
            continue
        for fname in os.listdir(county_dir):
            fpath = os.path.join(county_dir, fname)
            if os.path.isfile(fpath):
                try:
                    os.remove(fpath)
                    removed.append(f'{county["id"]}/{fname}')
                    gz_copy = os.path.join(GZIP_CACHE_DIR, county['id'], fname + '.gz')
                    if os.path.exists(gz_copy):
                        os.remove(gz_copy)
                except Exception as e:
                    errors.append(str(e))
    if errors:
        return jsonify({'error': '; '.join(errors)}), 500
    return jsonify({'success': True, 'removed': removed})

@app.route('/api/system-stats')
def system_stats():
    """Return current CPU, memory, and disk usage percentages."""
    if not _has_psutil:
        return jsonify({'error': 'psutil not installed'}), 503
    cpu = psutil.cpu_percent(interval=None)
    mem = psutil.virtual_memory().percent
    try:
        disk = psutil.disk_usage('/data').percent
    except Exception:
        disk = psutil.disk_usage('/').percent
    return jsonify({'cpu': cpu, 'mem': mem, 'disk': disk})

@app.route('/api/list-files')
def list_files():
    """List available GeoJSON files recursively with modification times."""
    data_dir = '/data'
    files = {'diff': [], 'osm': [], 'county': []}

    if os.path.exists(data_dir):
        for root, dirs, filenames in os.walk(data_dir):
            dirs[:] = [d for d in dirs if d != 'osm_cache']
            for filename in filenames:
                if filename.endswith('.geojson'):
                    filepath = os.path.join(root, filename)
                    rel_path = os.path.relpath(filepath, data_dir).replace('\\', '/')
                    mtime = datetime.fromtimestamp(os.path.getmtime(filepath)).strftime('%Y-%m-%d %H:%M')
                    entry = {'path': rel_path, 'mtime': mtime}
                    if 'diff' in filename.lower():
                        files['diff'].append(entry)
                    elif 'osm' in filename.lower():
                        files['osm'].append(entry)
                    else:
                        files['county'].append(entry)

    return jsonify(files)

GZIP_CACHE_DIR = '/data/.gzcache'
_gzip_lock = threading.Lock()


def _gzipped_copy(path, filename):
    """Path of a gzipped copy of path, (re)built when the source is newer.
    GeoJSON compresses ~10x, which matters for 50-150 MB county files."""
    gz_path = os.path.join(GZIP_CACHE_DIR, filename + '.gz')
    with _gzip_lock:
        if not os.path.exists(gz_path) or os.path.getmtime(gz_path) < os.path.getmtime(path):
            os.makedirs(os.path.dirname(gz_path), exist_ok=True)
            tmp = gz_path + '.tmp'
            with open(path, 'rb') as src, gzip.open(tmp, 'wb', compresslevel=5) as dst:
                shutil.copyfileobj(src, dst, 1024 * 1024)
            os.replace(tmp, gz_path)
    return gz_path


@app.route('/data/<path:filename>')
def serve_data(filename):
    """Serve files from /data (GeoJSON gzipped when the browser accepts it)"""
    path = safe_join('/data', filename)
    if (path and filename.endswith('.geojson') and os.path.isfile(path)
            and 'gzip' in request.headers.get('Accept-Encoding', '')):
        resp = make_response(send_file(_gzipped_copy(path, filename), mimetype='application/geo+json'))
        resp.headers['Content-Encoding'] = 'gzip'
        resp.headers['Vary'] = 'Accept-Encoding'
        return resp
    return send_from_directory('/data', filename)

# ── Config file download / upload ──────────────────────────────────────────────

_ALLOWED_CONFIGS = {'counties.yml', 'exceptions.yml', 'exclusions.yml'}

@app.route('/api/config/<filename>', methods=['GET'])
@require_auth
def download_config(filename):
    if filename not in _ALLOWED_CONFIGS:
        return jsonify({'error': 'Not found'}), 404
    return send_from_directory('/data', filename, as_attachment=True)

@app.route('/api/config/<filename>', methods=['POST'])
@require_auth
def upload_config(filename):
    if filename not in _ALLOWED_CONFIGS:
        return jsonify({'error': 'Not found'}), 404
    if 'file' not in request.files:
        return jsonify({'error': 'No file provided'}), 400
    f = request.files['file']
    try:
        content = f.read().decode('utf-8')
        data = yaml.safe_load(content)
        if filename == 'counties.yml' and not isinstance((data or {}).get('counties'), list):
            return jsonify({'error': 'Invalid counties.yml: missing counties list'}), 400
        if filename == 'exceptions.yml' and not isinstance((data or {}).get('corrections'), list):
            return jsonify({'error': 'Invalid exceptions.yml: missing corrections list'}), 400
        if filename == 'exclusions.yml' and not isinstance((data or {}).get('exclusions'), list):
            return jsonify({'error': 'Invalid exclusions.yml: missing exclusions list'}), 400
    except Exception as e:
        return jsonify({'error': f'Invalid YAML: {e}'}), 400
    dest = f'/data/{filename}'
    tmp = dest + '.tmp'
    with open(tmp, 'w') as out:
        out.write(content)
    os.replace(tmp, dest)
    return jsonify({'success': True})

# ── Exclusions config ─────────────────────────────────────────────────────────

EXCLUSIONS_FILE = '/data/exclusions.yml'


def _ensure_exclusions_file():
    if not os.path.exists(EXCLUSIONS_FILE):
        os.makedirs(os.path.dirname(EXCLUSIONS_FILE), exist_ok=True)
        with open(EXCLUSIONS_FILE, 'w') as f:
            yaml.dump({'exclusions': []}, f, default_flow_style=False, allow_unicode=True)

_ensure_exclusions_file()


def get_exclusions():
    try:
        with open(EXCLUSIONS_FILE) as f:
            data = yaml.safe_load(f) or {}
        return data.get('exclusions', [])
    except Exception:
        return []


@app.route('/exclusions')
def exclusions_page():
    return render_template('exclusions.html', current_user=session.get('username'))


@app.route('/api/exclusions', methods=['GET'])
def get_exclusions_api():
    return jsonify({'exclusions': get_exclusions()})


@app.route('/api/exclusions', methods=['POST'])
@require_auth
def save_exclusions():
    data = request.json
    items = data.get('exclusions', [])
    for item in items:
        if not isinstance(item.get('field'), str) or not item['field'].strip():
            return jsonify({'error': 'Each exclusion must have a non-empty "field"'}), 400
        if not isinstance(item.get('value'), str) or not item['value'].strip():
            return jsonify({'error': 'Each exclusion must have a non-empty "value"'}), 400
    tmp = EXCLUSIONS_FILE + '.tmp'
    with open(tmp, 'w') as f:
        yaml.dump({'exclusions': items}, f, default_flow_style=False, allow_unicode=True)
    os.replace(tmp, EXCLUSIONS_FILE)
    return jsonify({'success': True})


@app.route('/api/exclusions/add', methods=['POST'])
@require_auth
def add_exclusions():
    """Append new exclusion entries, skipping duplicates."""
    data = request.json
    new_items = data.get('exclusions', [])
    existing = get_exclusions()
    existing_keys = {(e['field'], e['value']) for e in existing}
    added = 0
    for item in new_items:
        field = (item.get('field') or '').strip()
        value = (item.get('value') or '').strip()
        if not field or not value:
            continue
        if (field, value) not in existing_keys:
            existing.append({'field': field, 'value': value})
            existing_keys.add((field, value))
            added += 1
    tmp = EXCLUSIONS_FILE + '.tmp'
    with open(tmp, 'w') as f:
        yaml.dump({'exclusions': existing}, f, default_flow_style=False, allow_unicode=True)
    os.replace(tmp, EXCLUSIONS_FILE)
    return jsonify({'success': True, 'added': added})

# ── Counties config ────────────────────────────────────────────────────────────

COUNTIES_FILE = '/data/counties.yml'

DEFAULT_COUNTIES = [
    {
        'id': 'sumter',
        'name': 'Sumter County',
        'state': 'Florida',
        'color': '#28a745',
        'roads_url': 'https://www.arcgis.com/sharing/rest/content/items/9177e17c72d3433aa79630c7eda84add/data',
        'addresses_url': 'https://www.arcgis.com/sharing/rest/content/items/c75c5aac13a648968c5596b0665be28b/data',
        'paths_url': 'upload',
        'pinned_cities': ['The Villages'],
    },
    {
        'id': 'lake',
        'name': 'Lake County',
        'state': 'Florida',
        'color': '#17a2b8',
        'roads_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Streets.zip',
        'addresses_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Addresspoints.zip',
        'paths_url': '',
        'pinned_cities': ['The Villages'],
    },
]

def _ensure_counties_file():
    if not os.path.exists(COUNTIES_FILE):
        os.makedirs(os.path.dirname(COUNTIES_FILE), exist_ok=True)
        with open(COUNTIES_FILE, 'w') as f:
            yaml.dump({'counties': DEFAULT_COUNTIES}, f,
                      default_flow_style=False, allow_unicode=True)

_ensure_counties_file()

def get_counties():
    try:
        with open(COUNTIES_FILE) as f:
            data = yaml.safe_load(f) or {}
        return data.get('counties', DEFAULT_COUNTIES)
    except Exception:
        return DEFAULT_COUNTIES

@app.route('/counties')
def counties_page():
    return render_template('counties.html', current_user=session.get('username'))

@app.route('/api/counties', methods=['GET'])
def get_counties_api():
    try:
        return jsonify({'counties': get_counties()})
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/api/counties', methods=['POST'])
@require_auth
def save_counties():
    data = request.json
    counties = data.get('counties', [])
    for c in counties:
        if not c.get('id', '').strip():
            return jsonify({'error': 'Each county must have a non-empty id'}), 400
        if not c.get('name', '').strip():
            return jsonify({'error': 'Each county must have a non-empty name'}), 400
    tmp = COUNTIES_FILE + '.tmp'
    with open(tmp, 'w') as f:
        yaml.dump({'counties': counties}, f, default_flow_style=False, allow_unicode=True)
    os.replace(tmp, COUNTIES_FILE)
    return jsonify({'success': True})

# ── Exceptions config ──────────────────────────────────────────────────────────

EXCEPTIONS_FILE = '/data/exceptions.yml'

DEFAULT_EXCEPTIONS = [
    {'from': "D Angelo Lane",        'to': "D'Angelo Lane"},
    {'from': "Lajolla Circle",       'to': "La Jolla Circle"},
    {'from': "Gardena Court",        'to': "Gardenia Court"},
    {'from': "Glenmont Court",       'to': "Glenmount Court"},
    {'from': "Pawleys Island Path",  'to': "Pawley's Island Path"},
    {'from': "Oday Street",          'to': "O'Day Street"},
    {'from': "Obrien Place",         'to': "O'Brien Place"},
    {'from': "Ohara Court",          'to': "O'Hara Court"},
]

def _ensure_exceptions_file():
    if not os.path.exists(EXCEPTIONS_FILE):
        os.makedirs(os.path.dirname(EXCEPTIONS_FILE), exist_ok=True)
        with open(EXCEPTIONS_FILE, 'w') as f:
            yaml.dump({'corrections': DEFAULT_EXCEPTIONS}, f,
                      default_flow_style=False, allow_unicode=True)

_ensure_exceptions_file()


@app.route('/exceptions')
def exceptions_page():
    return render_template('exceptions.html', current_user=session.get('username'))


@app.route('/api/exceptions', methods=['GET'])
def get_exceptions():
    try:
        with open(EXCEPTIONS_FILE) as f:
            data = yaml.safe_load(f) or {}
        return jsonify({'corrections': data.get('corrections', [])})
    except Exception as e:
        return jsonify({'error': str(e)}), 500


@app.route('/api/exceptions', methods=['POST'])
@require_auth
def save_exceptions():
    data = request.json
    corrections = data.get('corrections', [])
    for item in corrections:
        if not isinstance(item.get('from'), str) or not item['from'].strip():
            return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400
        if not isinstance(item.get('to'), str) or not item['to'].strip():
            return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400
    tmp = EXCEPTIONS_FILE + '.tmp'
    with open(tmp, 'w') as f:
        yaml.dump({'corrections': corrections}, f,
                  default_flow_style=False, allow_unicode=True)
    os.replace(tmp, EXCEPTIONS_FILE)
    return jsonify({'success': True})


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=True)