osm-import-tools/web/server.py

#!/usr/bin/env python3
"""
Flask web server for The Villages Import Tools
"""
from flask import Flask, render_template, jsonify, request, send_from_directory
import subprocess
import os
import threading
import json
from datetime import datetime
import yaml

app = Flask(__name__, static_folder='static', template_folder='templates')

# Store running processes
running_processes = {}
process_logs = {}

# Global var
latest_path = "/data/latest"

@app.route('/')
def index():
    """Main index page with script execution buttons"""
    # Get available scripts and organize them
    script_map = get_script_map()

    # Organize scripts by category
    scripts_by_category = {
        'Download County Data': ['download-county-addresses', 'download-county-roads', 'download-county-paths'],
        'Download OSM Data': ['download-osm-roads', 'download-osm-addresses', 'download-osm-paths'],
        'Convert Data': ['convert-roads', 'convert-addresses', 'convert-paths'],
        'Diff Data': ['diff-roads', 'diff-paths', 'diff-addresses'],
        'Utilities': ['ls', 'make-new-latest']
    }

    # Get list of files
    data_files = {}
    if os.path.exists(latest_path):
        for countyFolder in os.listdir(latest_path):
            files = os.listdir('/data/latest/'+countyFolder)
            data_files[countyFolder] = files
            # data_files.append(countyFolder)

    return render_template('index.html',
                         script_map=script_map,
                         scripts_by_category=scripts_by_category,
                         data_files=data_files
                         )

@app.route('/map')
def map_viewer():
    """Map viewer page"""
    return render_template('map.html')

def get_script_map():
    """Get the map of available scripts and their commands"""
    return {
        'ls': 'ls -al /data',
        'make-new-latest': 'cd /data && NEWDIR=$(date +%y%m%d) && mkdir -p $NEWDIR/lake $NEWDIR/sumter && ln -sfn $NEWDIR latest',
        # todo: make a clean-old-data script
        'download-county-paths': {
            'sumter': {'type': 'upload', 'dest': '/data/latest/sumter/paths.shp.zip'},
            # lake: not available
        },
        'download-county-addresses': {
            # deliver files with standardized names
            'sumter': 'mkdir -p /data/latest/sumter && wget https://www.arcgis.com/sharing/rest/content/items/c75c5aac13a648968c5596b0665be28b/data -O /data/latest/sumter/addresses.shp.zip',
            'lake': 'mkdir -p /data/latest/lake && wget https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Addresspoints.zip -O /data/latest/lake/addresses.shp.zip'
        },
        'download-county-roads': {
            # deliver files with standardized names
            'sumter': 'mkdir -p /data/latest/sumter && wget https://www.arcgis.com/sharing/rest/content/items/9177e17c72d3433aa79630c7eda84add/data -O /data/latest/sumter/roads.shp.zip',
            'lake': 'mkdir -p /data/latest/lake && wget https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Streets.zip -O /data/latest/lake/roads.shp.zip'
        },
        # todo: integrate osm downloading and shapefile converting into diff-roads like addresses
        'download-osm-roads': {
            'lake': ['python', 'download-overpass.py', '--type', 'highways', 'Lake County', 'Florida', '/data/latest/lake/osm-roads.geojson'],
            'sumter': ['python', 'download-overpass.py', '--type', 'highways', 'Sumter County', 'Florida', '/data/latest/sumter/osm-roads.geojson']
        },
        'download-osm-addresses': {
            'lake': ['python', 'download-overpass.py', '--type', 'addresses', 'Lake County', 'Florida', '/data/latest/lake/osm-addresses.geojson'],
            'sumter': ['python', 'download-overpass.py', '--type', 'addresses', 'Sumter County', 'Florida', '/data/latest/sumter/osm-addresses.geojson']
        },
        'download-osm-paths': {
            # todo: no lake county paths
            #'lake': ['python', 'download-overpass.py', '--type', 'highways', 'Lake County', 'Florida', '/data/latest/lake/osm-roads.geojson'],
            'sumter': ['python', 'download-overpass.py', '--type', 'paths', 'Sumter County', 'Florida', '/data/latest/sumter/osm-paths.geojson']
        },
        # todo
        'convert-roads': {
            'sumter': ['python', 'shp-to-geojson.py', '/data/latest/sumter/roads.shp.zip', '/data/latest/sumter/county-roads.geojson'],
            'lake': ['python', 'shp-to-geojson.py', '/data/latest/lake/roads.shp.zip', '/data/latest/lake/county-roads.geojson']
        },
        'convert-addresses': {
            'sumter': ['python', 'convert-addresses.py', '/data/latest/sumter/addresses.shp.zip', '/data/latest/sumter/county-addresses.geojson'],
            'lake':   ['python', 'convert-addresses.py', '/data/latest/lake/addresses.shp.zip',   '/data/latest/lake/county-addresses.geojson'],
        },
        'convert-paths': {
            'sumter': ['python', 'shp-to-geojson.py', '/data/latest/sumter/paths.shp.zip', '/data/latest/sumter/county-paths.geojson'],
        },
        'diff-roads': {
            'lake':   ['python', 'diff-highways.py', '/data/latest/lake/osm-roads.geojson',   '/data/latest/lake/county-roads.geojson',   '--output', '/data/latest/lake/diff-roads.geojson'],
            'sumter': ['python', 'diff-highways.py', '/data/latest/sumter/osm-roads.geojson', '/data/latest/sumter/county-roads.geojson', '--output', '/data/latest/sumter/diff-roads.geojson'],
        },
        'diff-paths': {
            'sumter': ['python', 'diff-highways.py', '/data/latest/sumter/osm-paths.geojson', '/data/latest/sumter/county-paths.geojson', '--output', '/data/latest/sumter/diff-paths.geojson'],
        },
        'diff-addresses': {
            'lake':   ['python', 'compare-addresses.py', '--local-file', '/data/latest/lake/county-addresses.geojson',   '--osm-file', '/data/latest/lake/osm-addresses.geojson',   '--output-dir', '/data/latest/lake'],
            'sumter': ['python', 'compare-addresses.py', '--local-file', '/data/latest/sumter/county-addresses.geojson', '--osm-file', '/data/latest/sumter/osm-addresses.geojson', '--output-dir', '/data/latest/sumter'],
        },
    }

@app.route('/api/run-script', methods=['POST'])
def run_script():
    """Execute a script in the background"""
    data = request.json
    script_name = data.get('script')
    county = data.get('county', '')
    force_download = data.get('forceDownload', False)

    if not script_name:
        return jsonify({'error': 'No script specified'}), 400

    # Check if /data/latest exists and is a symlink, not a regular directory
    # Skip this check for make-new-latest, ls
    skip_check_scripts = ['make-new-latest', 'ls']
    if script_name not in skip_check_scripts:
        latest_path = '/data/latest'
        if os.path.exists(latest_path) or os.path.islink(latest_path):
            # Check if it's a directory but NOT a symlink
            if os.path.isdir(latest_path) and not os.path.islink(latest_path):
                return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400
        else:
            # /data/latest doesn't exist at all
            return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400

    script_map = get_script_map()

    if script_name not in script_map:
        return jsonify({'error': 'Unknown script'}), 400

    script_config = script_map[script_name]

    # Handle both string commands and dict of county-specific commands
    if isinstance(script_config, str):
        # Simple string command (like 'ls')
        cmd = ['bash', '-c', script_config]
    elif isinstance(script_config, dict):
        # County-specific commands
        if not county:
            return jsonify({'error': 'County required for this script'}), 400
        if county not in script_config:
            return jsonify({'error': f'County {county} not supported for {script_name}'}), 400

        cmd_config = script_config[county]
        if isinstance(cmd_config, str):
            cmd = ['bash', '-c', cmd_config]
        else:
            cmd = list(cmd_config)  # Make a copy to avoid modifying the original

    else:
        return jsonify({'error': 'Invalid script configuration'}), 400

    # Generate a unique job ID
    job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"

    # Start process in background
    def run_command():
        try:
            process = subprocess.Popen(
                cmd,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                text=True,
                cwd=os.path.dirname(os.path.dirname(__file__))
            )
            running_processes[job_id] = process
            process_logs[job_id] = []

            # Stream output
            for line in process.stdout:
                process_logs[job_id].append(line)

            process.wait()
            process_logs[job_id].append(f"\n[Process completed with exit code {process.returncode}]")

        except Exception as e:
            process_logs[job_id].append(f"\n[ERROR: {str(e)}]")
        finally:
            if job_id in running_processes:
                del running_processes[job_id]

    thread = threading.Thread(target=run_command)
    thread.daemon = True
    thread.start()

    return jsonify({'job_id': job_id, 'status': 'started'})

@app.route('/api/job-status/<job_id>')
def job_status(job_id):
    """Get status and logs for a job"""
    is_running = job_id in running_processes
    logs = process_logs.get(job_id, [])

    return jsonify({
        'job_id': job_id,
        'running': is_running,
        'logs': logs
    })

@app.route('/api/cancel-job', methods=['POST'])
def cancel_job():
    """Cancel a running job"""
    data = request.json
    job_id = data.get('job_id')

    if not job_id:
        return jsonify({'error': 'No job ID specified'}), 400

    if job_id not in running_processes:
        return jsonify({'error': 'Job not found or already completed'}), 404

    try:
        process = running_processes[job_id]
        process.terminate()
        process_logs[job_id].append('\n[Job cancelled by user]')
        del running_processes[job_id]
        return jsonify({'success': True})
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/api/upload-file', methods=['POST'])
def upload_file():
    """Upload a shapefile ZIP to a path under /data/latest"""
    dest = request.form.get('dest')
    if not dest or not dest.startswith('/data/latest/'):
        return jsonify({'error': 'Invalid destination path'}), 400

    latest = '/data/latest'
    if os.path.exists(latest) or os.path.islink(latest):
        if os.path.isdir(latest) and not os.path.islink(latest):
            return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400
    else:
        return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400

    if 'file' not in request.files:
        return jsonify({'error': 'No file provided'}), 400

    f = request.files['file']
    if not f.filename.lower().endswith('.zip'):
        return jsonify({'error': 'File must be a ZIP archive (.zip)'}), 400

    os.makedirs(os.path.dirname(dest), exist_ok=True)
    f.save(dest)
    return jsonify({'success': True, 'path': dest})

@app.route('/api/list-files')
def list_files():
    """List available GeoJSON files"""
    data_dir = '/data'

    files = {
        'diff': [],
        'osm': [],
        'county': []
    }

    # Scan directories for geojson files
    if os.path.exists(data_dir):
        for root, dirs, filenames in os.walk(data_dir):
            for filename in filenames:
                if filename.endswith('.geojson'):
                    rel_path = os.path.relpath(os.path.join(root, filename), data_dir)

                    if 'diff' in filename.lower():
                        files['diff'].append(rel_path)
                    elif 'osm' in filename.lower():
                        files['osm'].append(rel_path)
                    elif any(county in filename.lower() for county in ['lake', 'sumter']):
                        files['county'].append(rel_path)

    return jsonify(files)

@app.route('/data/<path:filename>')
def serve_data(filename):
    """Serve GeoJSON files"""
    return send_from_directory('/data', filename)

EXCEPTIONS_FILE = '/data/exceptions.yml'

DEFAULT_EXCEPTIONS = [
    {'from': "D Angelo Lane",        'to': "D'Angelo Lane"},
    {'from': "Lajolla Circle",       'to': "La Jolla Circle"},
    {'from': "Gardena Court",        'to': "Gardenia Court"},
    {'from': "Glenmont Court",       'to': "Glenmount Court"},
    {'from': "Pawleys Island Path",  'to': "Pawley's Island Path"},
    {'from': "Oday Street",          'to': "O'Day Street"},
    {'from': "Obrien Place",         'to': "O'Brien Place"},
    {'from': "Ohara Court",          'to': "O'Hara Court"},
]

def _ensure_exceptions_file():
    if not os.path.exists(EXCEPTIONS_FILE):
        os.makedirs(os.path.dirname(EXCEPTIONS_FILE), exist_ok=True)
        with open(EXCEPTIONS_FILE, 'w') as f:
            yaml.dump({'corrections': DEFAULT_EXCEPTIONS}, f,
                      default_flow_style=False, allow_unicode=True)

_ensure_exceptions_file()


@app.route('/exceptions')
def exceptions_page():
    return render_template('exceptions.html')


@app.route('/api/exceptions', methods=['GET'])
def get_exceptions():
    try:
        with open(EXCEPTIONS_FILE) as f:
            data = yaml.safe_load(f) or {}
        return jsonify({'corrections': data.get('corrections', [])})
    except Exception as e:
        return jsonify({'error': str(e)}), 500


@app.route('/api/exceptions', methods=['POST'])
def save_exceptions():
    data = request.json
    corrections = data.get('corrections', [])

    # Validate
    for item in corrections:
        if not isinstance(item.get('from'), str) or not item['from'].strip():
            return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400
        if not isinstance(item.get('to'), str) or not item['to'].strip():
            return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400

    # Atomic write
    tmp = EXCEPTIONS_FILE + '.tmp'
    with open(tmp, 'w') as f:
        yaml.dump({'corrections': corrections}, f,
                  default_flow_style=False, allow_unicode=True)
    os.replace(tmp, EXCEPTIONS_FILE)
    return jsonify({'success': True})


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=True)