zyphlar
482 lines
18 KiB
Python
482 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Flask web server for The Villages Import Tools
|
|
"""
|
|
from flask import Flask, render_template, jsonify, request, send_from_directory
|
|
import subprocess
|
|
import os
|
|
import re
|
|
import shutil
|
|
import threading
|
|
import json
|
|
from datetime import datetime
|
|
import yaml
|
|
try:
|
|
import psutil
|
|
_has_psutil = True
|
|
except ImportError:
|
|
_has_psutil = False
|
|
|
|
app = Flask(__name__, static_folder='static', template_folder='templates')
|
|
|
|
# Store running processes
|
|
running_processes = {}
|
|
process_logs = {}
|
|
|
|
|
|
def _cleanup_legacy_structure():
|
|
"""Remove dated folders and symlink from the old /data/YYMMDD layout,
|
|
migrating any files up to /data/{county}/ before deleting."""
|
|
data_dir = '/data'
|
|
if not os.path.exists(data_dir):
|
|
return
|
|
|
|
# Remove the legacy 'latest' symlink
|
|
symlink = os.path.join(data_dir, 'latest')
|
|
if os.path.islink(symlink):
|
|
os.unlink(symlink)
|
|
elif os.path.isdir(symlink):
|
|
# Safety: if it somehow became a real dir, leave it alone
|
|
pass
|
|
|
|
# Migrate and remove 6-digit dated folders (YYMMDD)
|
|
dated_pattern = re.compile(r'^\d{6}$')
|
|
for entry in os.listdir(data_dir):
|
|
entry_path = os.path.join(data_dir, entry)
|
|
if os.path.isdir(entry_path) and dated_pattern.match(entry):
|
|
for county_dir in os.listdir(entry_path):
|
|
src_county = os.path.join(entry_path, county_dir)
|
|
if os.path.isdir(src_county):
|
|
dst_county = os.path.join(data_dir, county_dir)
|
|
os.makedirs(dst_county, exist_ok=True)
|
|
for fname in os.listdir(src_county):
|
|
src_file = os.path.join(src_county, fname)
|
|
dst_file = os.path.join(dst_county, fname)
|
|
if os.path.isfile(src_file) and not os.path.exists(dst_file):
|
|
shutil.move(src_file, dst_file)
|
|
shutil.rmtree(entry_path)
|
|
|
|
_cleanup_legacy_structure()
|
|
|
|
|
|
@app.route('/')
|
|
def index():
|
|
"""Main index page with script execution buttons"""
|
|
counties = get_counties()
|
|
|
|
# Get list of files with modification times
|
|
data_files = {}
|
|
for county in counties:
|
|
county_dir = f'/data/{county["id"]}'
|
|
if os.path.isdir(county_dir):
|
|
files = []
|
|
for fname in sorted(os.listdir(county_dir)):
|
|
fpath = os.path.join(county_dir, fname)
|
|
if os.path.isfile(fpath):
|
|
mtime = datetime.fromtimestamp(os.path.getmtime(fpath)).strftime('%Y-%m-%d %H:%M')
|
|
files.append({'name': fname, 'mtime': mtime})
|
|
if files:
|
|
data_files[county['id']] = files
|
|
|
|
import json as _json
|
|
return render_template('index.html',
|
|
counties=counties,
|
|
counties_json=_json.dumps(counties),
|
|
data_files=data_files)
|
|
|
|
@app.route('/map')
|
|
def map_viewer():
|
|
"""Map viewer page"""
|
|
return render_template('map.html')
|
|
|
|
def get_script_map():
|
|
"""Build script map dynamically from county configuration."""
|
|
counties = get_counties()
|
|
|
|
sm = {
|
|
'ls': 'ls -alR /data',
|
|
}
|
|
|
|
dl_roads = {}; dl_addrs = {}; dl_paths = {}
|
|
osm_roads = {}; osm_addrs = {}; osm_paths = {}
|
|
conv_roads = {}; conv_addrs = {}; conv_paths = {}
|
|
diff_roads = {}; diff_addrs = {}; diff_paths = {}
|
|
|
|
for c in counties:
|
|
cid = c['id']
|
|
name = c['name']
|
|
state = c['state']
|
|
base = f'/data/{cid}'
|
|
|
|
if c.get('roads_url'):
|
|
dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip'
|
|
if c.get('addresses_url'):
|
|
dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip'
|
|
pu = c.get('paths_url', '')
|
|
if pu == 'upload':
|
|
dl_paths[cid] = {'type': 'upload', 'dest': f'{base}/paths.shp.zip'}
|
|
elif pu:
|
|
dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip'
|
|
|
|
osm_roads[cid] = ['python', 'download-overpass.py', '--type', 'highways', name, state, f'{base}/osm-roads.geojson']
|
|
osm_addrs[cid] = ['python', 'download-overpass.py', '--type', 'addresses', name, state, f'{base}/osm-addresses.geojson']
|
|
if pu:
|
|
osm_paths[cid] = ['python', 'download-overpass.py', '--type', 'paths', name, state, f'{base}/osm-paths.geojson']
|
|
|
|
conv_roads[cid] = ['python', 'shp-to-geojson.py', f'{base}/roads.shp.zip', f'{base}/county-roads.geojson']
|
|
conv_addrs[cid] = ['python', 'convert-addresses.py', f'{base}/addresses.shp.zip', f'{base}/county-addresses.geojson']
|
|
if pu:
|
|
conv_paths[cid] = ['python', 'shp-to-geojson.py', f'{base}/paths.shp.zip', f'{base}/county-paths.geojson']
|
|
|
|
diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson']
|
|
diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base]
|
|
if pu:
|
|
diff_paths[cid] = ['python', 'diff-highways.py', f'{base}/osm-paths.geojson', f'{base}/county-paths.geojson', '--output', f'{base}/diff-paths.geojson']
|
|
|
|
if dl_roads: sm['download-county-roads'] = dl_roads
|
|
if dl_addrs: sm['download-county-addresses'] = dl_addrs
|
|
if dl_paths: sm['download-county-paths'] = dl_paths
|
|
sm['download-osm-roads'] = osm_roads
|
|
sm['download-osm-addresses'] = osm_addrs
|
|
if osm_paths: sm['download-osm-paths'] = osm_paths
|
|
sm['convert-roads'] = conv_roads
|
|
sm['convert-addresses'] = conv_addrs
|
|
if conv_paths: sm['convert-paths'] = conv_paths
|
|
sm['diff-roads'] = diff_roads
|
|
sm['diff-addresses'] = diff_addrs
|
|
if diff_paths: sm['diff-paths'] = diff_paths
|
|
|
|
return sm
|
|
|
|
@app.route('/api/run-script', methods=['POST'])
|
|
def run_script():
|
|
"""Execute a script in the background"""
|
|
data = request.json
|
|
script_name = data.get('script')
|
|
county = data.get('county', '')
|
|
force_download = data.get('forceDownload', False)
|
|
|
|
if not script_name:
|
|
return jsonify({'error': 'No script specified'}), 400
|
|
|
|
script_map = get_script_map()
|
|
|
|
if script_name not in script_map:
|
|
return jsonify({'error': 'Unknown script'}), 400
|
|
|
|
script_config = script_map[script_name]
|
|
|
|
# Handle both string commands and dict of county-specific commands
|
|
if isinstance(script_config, str):
|
|
cmd = ['bash', '-c', script_config]
|
|
elif isinstance(script_config, dict):
|
|
if not county:
|
|
return jsonify({'error': 'County required for this script'}), 400
|
|
if county not in script_config:
|
|
return jsonify({'error': f'County {county} not supported for {script_name}'}), 400
|
|
|
|
cmd_config = script_config[county]
|
|
if isinstance(cmd_config, str):
|
|
cmd = ['bash', '-c', cmd_config]
|
|
else:
|
|
cmd = list(cmd_config)
|
|
|
|
else:
|
|
return jsonify({'error': 'Invalid script configuration'}), 400
|
|
|
|
# Generate a unique job ID
|
|
job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
|
|
|
# Start process in background
|
|
def run_command():
|
|
try:
|
|
process = subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
cwd=os.path.dirname(os.path.dirname(__file__))
|
|
)
|
|
running_processes[job_id] = process
|
|
process_logs[job_id] = []
|
|
|
|
for line in process.stdout:
|
|
process_logs[job_id].append(line)
|
|
|
|
process.wait()
|
|
process_logs[job_id].append(f"\n[Process completed with exit code {process.returncode}]")
|
|
|
|
except Exception as e:
|
|
process_logs[job_id].append(f"\n[ERROR: {str(e)}]")
|
|
finally:
|
|
if job_id in running_processes:
|
|
del running_processes[job_id]
|
|
|
|
thread = threading.Thread(target=run_command)
|
|
thread.daemon = True
|
|
thread.start()
|
|
|
|
return jsonify({'job_id': job_id, 'status': 'started'})
|
|
|
|
@app.route('/api/job-status/<job_id>')
|
|
def job_status(job_id):
|
|
"""Get status and logs for a job"""
|
|
is_running = job_id in running_processes
|
|
logs = process_logs.get(job_id, [])
|
|
|
|
return jsonify({
|
|
'job_id': job_id,
|
|
'running': is_running,
|
|
'logs': logs
|
|
})
|
|
|
|
@app.route('/api/cancel-job', methods=['POST'])
|
|
def cancel_job():
|
|
"""Cancel a running job"""
|
|
data = request.json
|
|
job_id = data.get('job_id')
|
|
|
|
if not job_id:
|
|
return jsonify({'error': 'No job ID specified'}), 400
|
|
|
|
if job_id not in running_processes:
|
|
return jsonify({'error': 'Job not found or already completed'}), 404
|
|
|
|
try:
|
|
process = running_processes[job_id]
|
|
process.terminate()
|
|
process_logs[job_id].append('\n[Job cancelled by user]')
|
|
del running_processes[job_id]
|
|
return jsonify({'success': True})
|
|
except Exception as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
@app.route('/api/upload-file', methods=['POST'])
|
|
def upload_file():
|
|
"""Upload a shapefile ZIP to a path under /data"""
|
|
dest = request.form.get('dest')
|
|
if not dest or not dest.startswith('/data/'):
|
|
return jsonify({'error': 'Invalid destination path'}), 400
|
|
|
|
if 'file' not in request.files:
|
|
return jsonify({'error': 'No file provided'}), 400
|
|
|
|
f = request.files['file']
|
|
if not f.filename.lower().endswith('.zip'):
|
|
return jsonify({'error': 'File must be a ZIP archive (.zip)'}), 400
|
|
|
|
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
f.save(dest)
|
|
return jsonify({'success': True, 'path': dest})
|
|
|
|
@app.route('/api/system-stats')
|
|
def system_stats():
|
|
"""Return current CPU, memory, and disk usage percentages."""
|
|
if not _has_psutil:
|
|
return jsonify({'error': 'psutil not installed'}), 503
|
|
cpu = psutil.cpu_percent(interval=None)
|
|
mem = psutil.virtual_memory().percent
|
|
try:
|
|
disk = psutil.disk_usage('/data').percent
|
|
except Exception:
|
|
disk = psutil.disk_usage('/').percent
|
|
return jsonify({'cpu': cpu, 'mem': mem, 'disk': disk})
|
|
|
|
@app.route('/api/list-files')
|
|
def list_files():
|
|
"""List available GeoJSON files recursively with modification times."""
|
|
data_dir = '/data'
|
|
|
|
files = {
|
|
'diff': [],
|
|
'osm': [],
|
|
'county': []
|
|
}
|
|
|
|
if os.path.exists(data_dir):
|
|
for root, dirs, filenames in os.walk(data_dir):
|
|
dirs[:] = [d for d in dirs if d != 'osm_cache']
|
|
for filename in filenames:
|
|
if filename.endswith('.geojson'):
|
|
filepath = os.path.join(root, filename)
|
|
rel_path = os.path.relpath(filepath, data_dir).replace('\\', '/')
|
|
mtime = datetime.fromtimestamp(os.path.getmtime(filepath)).strftime('%Y-%m-%d %H:%M')
|
|
entry = {'path': rel_path, 'mtime': mtime}
|
|
|
|
if 'diff' in filename.lower():
|
|
files['diff'].append(entry)
|
|
elif 'osm' in filename.lower():
|
|
files['osm'].append(entry)
|
|
else:
|
|
files['county'].append(entry)
|
|
|
|
return jsonify(files)
|
|
|
|
@app.route('/data/<path:filename>')
|
|
def serve_data(filename):
|
|
"""Serve files from /data"""
|
|
return send_from_directory('/data', filename)
|
|
|
|
# ── Config file download / upload ──────────────────────────────────────────────
|
|
|
|
_ALLOWED_CONFIGS = {'counties.yml', 'exceptions.yml'}
|
|
|
|
@app.route('/api/config/<filename>', methods=['GET'])
|
|
def download_config(filename):
|
|
if filename not in _ALLOWED_CONFIGS:
|
|
return jsonify({'error': 'Not found'}), 404
|
|
return send_from_directory('/data', filename, as_attachment=True)
|
|
|
|
@app.route('/api/config/<filename>', methods=['POST'])
|
|
def upload_config(filename):
|
|
if filename not in _ALLOWED_CONFIGS:
|
|
return jsonify({'error': 'Not found'}), 404
|
|
if 'file' not in request.files:
|
|
return jsonify({'error': 'No file provided'}), 400
|
|
f = request.files['file']
|
|
try:
|
|
content = f.read().decode('utf-8')
|
|
data = yaml.safe_load(content)
|
|
if filename == 'counties.yml' and not isinstance((data or {}).get('counties'), list):
|
|
return jsonify({'error': 'Invalid counties.yml: missing counties list'}), 400
|
|
if filename == 'exceptions.yml' and not isinstance((data or {}).get('corrections'), list):
|
|
return jsonify({'error': 'Invalid exceptions.yml: missing corrections list'}), 400
|
|
except Exception as e:
|
|
return jsonify({'error': f'Invalid YAML: {e}'}), 400
|
|
dest = f'/data/{filename}'
|
|
tmp = dest + '.tmp'
|
|
with open(tmp, 'w') as out:
|
|
out.write(content)
|
|
os.replace(tmp, dest)
|
|
return jsonify({'success': True})
|
|
|
|
# ── Counties config ────────────────────────────────────────────────────────────
|
|
|
|
COUNTIES_FILE = '/data/counties.yml'
|
|
|
|
DEFAULT_COUNTIES = [
|
|
{
|
|
'id': 'sumter',
|
|
'name': 'Sumter County',
|
|
'state': 'Florida',
|
|
'color': '#28a745',
|
|
'roads_url': 'https://www.arcgis.com/sharing/rest/content/items/9177e17c72d3433aa79630c7eda84add/data',
|
|
'addresses_url': 'https://www.arcgis.com/sharing/rest/content/items/c75c5aac13a648968c5596b0665be28b/data',
|
|
'paths_url': 'upload',
|
|
},
|
|
{
|
|
'id': 'lake',
|
|
'name': 'Lake County',
|
|
'state': 'Florida',
|
|
'color': '#17a2b8',
|
|
'roads_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Streets.zip',
|
|
'addresses_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Addresspoints.zip',
|
|
'paths_url': '',
|
|
},
|
|
]
|
|
|
|
def _ensure_counties_file():
|
|
if not os.path.exists(COUNTIES_FILE):
|
|
os.makedirs(os.path.dirname(COUNTIES_FILE), exist_ok=True)
|
|
with open(COUNTIES_FILE, 'w') as f:
|
|
yaml.dump({'counties': DEFAULT_COUNTIES}, f,
|
|
default_flow_style=False, allow_unicode=True)
|
|
|
|
_ensure_counties_file()
|
|
|
|
def get_counties():
|
|
try:
|
|
with open(COUNTIES_FILE) as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return data.get('counties', DEFAULT_COUNTIES)
|
|
except Exception:
|
|
return DEFAULT_COUNTIES
|
|
|
|
@app.route('/counties')
|
|
def counties_page():
|
|
return render_template('counties.html')
|
|
|
|
@app.route('/api/counties', methods=['GET'])
|
|
def get_counties_api():
|
|
try:
|
|
return jsonify({'counties': get_counties()})
|
|
except Exception as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
@app.route('/api/counties', methods=['POST'])
|
|
def save_counties():
|
|
data = request.json
|
|
counties = data.get('counties', [])
|
|
for c in counties:
|
|
if not c.get('id', '').strip():
|
|
return jsonify({'error': 'Each county must have a non-empty id'}), 400
|
|
if not c.get('name', '').strip():
|
|
return jsonify({'error': 'Each county must have a non-empty name'}), 400
|
|
tmp = COUNTIES_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
yaml.dump({'counties': counties}, f, default_flow_style=False, allow_unicode=True)
|
|
os.replace(tmp, COUNTIES_FILE)
|
|
return jsonify({'success': True})
|
|
|
|
# ── Exceptions config ──────────────────────────────────────────────────────────
|
|
|
|
EXCEPTIONS_FILE = '/data/exceptions.yml'
|
|
|
|
DEFAULT_EXCEPTIONS = [
|
|
{'from': "D Angelo Lane", 'to': "D'Angelo Lane"},
|
|
{'from': "Lajolla Circle", 'to': "La Jolla Circle"},
|
|
{'from': "Gardena Court", 'to': "Gardenia Court"},
|
|
{'from': "Glenmont Court", 'to': "Glenmount Court"},
|
|
{'from': "Pawleys Island Path", 'to': "Pawley's Island Path"},
|
|
{'from': "Oday Street", 'to': "O'Day Street"},
|
|
{'from': "Obrien Place", 'to': "O'Brien Place"},
|
|
{'from': "Ohara Court", 'to': "O'Hara Court"},
|
|
]
|
|
|
|
def _ensure_exceptions_file():
|
|
if not os.path.exists(EXCEPTIONS_FILE):
|
|
os.makedirs(os.path.dirname(EXCEPTIONS_FILE), exist_ok=True)
|
|
with open(EXCEPTIONS_FILE, 'w') as f:
|
|
yaml.dump({'corrections': DEFAULT_EXCEPTIONS}, f,
|
|
default_flow_style=False, allow_unicode=True)
|
|
|
|
_ensure_exceptions_file()
|
|
|
|
|
|
@app.route('/exceptions')
|
|
def exceptions_page():
|
|
return render_template('exceptions.html')
|
|
|
|
|
|
@app.route('/api/exceptions', methods=['GET'])
|
|
def get_exceptions():
|
|
try:
|
|
with open(EXCEPTIONS_FILE) as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return jsonify({'corrections': data.get('corrections', [])})
|
|
except Exception as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@app.route('/api/exceptions', methods=['POST'])
|
|
def save_exceptions():
|
|
data = request.json
|
|
corrections = data.get('corrections', [])
|
|
|
|
for item in corrections:
|
|
if not isinstance(item.get('from'), str) or not item['from'].strip():
|
|
return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400
|
|
if not isinstance(item.get('to'), str) or not item['to'].strip():
|
|
return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400
|
|
|
|
tmp = EXCEPTIONS_FILE + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
yaml.dump({'corrections': corrections}, f,
|
|
default_flow_style=False, allow_unicode=True)
|
|
os.replace(tmp, EXCEPTIONS_FILE)
|
|
return jsonify({'success': True})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
app.run(host='0.0.0.0', port=5000, debug=True)
|