diff --git a/web/server.py b/web/server.py index b4b5b7e..7e94281 100644 --- a/web/server.py +++ b/web/server.py @@ -5,6 +5,8 @@ Flask web server for The Villages Import Tools from flask import Flask, render_template, jsonify, request, send_from_directory import subprocess import os +import re +import shutil import threading import json from datetime import datetime @@ -21,24 +23,60 @@ app = Flask(__name__, static_folder='static', template_folder='templates') running_processes = {} process_logs = {} -# Global var -latest_path = "/data/latest" + +def _cleanup_legacy_structure(): + """Remove dated folders and symlink from the old /data/YYMMDD layout, + migrating any files up to /data/{county}/ before deleting.""" + data_dir = '/data' + if not os.path.exists(data_dir): + return + + # Remove the legacy 'latest' symlink + symlink = os.path.join(data_dir, 'latest') + if os.path.islink(symlink): + os.unlink(symlink) + elif os.path.isdir(symlink): + # Safety: if it somehow became a real dir, leave it alone + pass + + # Migrate and remove 6-digit dated folders (YYMMDD) + dated_pattern = re.compile(r'^\d{6}$') + for entry in os.listdir(data_dir): + entry_path = os.path.join(data_dir, entry) + if os.path.isdir(entry_path) and dated_pattern.match(entry): + for county_dir in os.listdir(entry_path): + src_county = os.path.join(entry_path, county_dir) + if os.path.isdir(src_county): + dst_county = os.path.join(data_dir, county_dir) + os.makedirs(dst_county, exist_ok=True) + for fname in os.listdir(src_county): + src_file = os.path.join(src_county, fname) + dst_file = os.path.join(dst_county, fname) + if os.path.isfile(src_file) and not os.path.exists(dst_file): + shutil.move(src_file, dst_file) + shutil.rmtree(entry_path) + +_cleanup_legacy_structure() + @app.route('/') def index(): """Main index page with script execution buttons""" counties = get_counties() - # Get list of files + # Get list of files with modification times data_files = {} - if os.path.exists(latest_path): - try: - for countyFolder in os.listdir(latest_path): - folder_path = '/data/latest/' + countyFolder - if os.path.isdir(folder_path): - data_files[countyFolder] = os.listdir(folder_path) - except Exception: - pass + for county in counties: + county_dir = f'/data/{county["id"]}' + if os.path.isdir(county_dir): + files = [] + for fname in sorted(os.listdir(county_dir)): + fpath = os.path.join(county_dir, fname) + if os.path.isfile(fpath): + mtime = datetime.fromtimestamp(os.path.getmtime(fpath)).strftime('%Y-%m-%d %H:%M') + files.append({'name': fname, 'mtime': mtime}) + if files: + data_files[county['id']] = files import json as _json return render_template('index.html', @@ -55,10 +93,8 @@ def get_script_map(): """Build script map dynamically from county configuration.""" counties = get_counties() - county_dirs = ' '.join(f'$NEWDIR/{c["id"]}' for c in counties) sm = { - 'ls': 'ls -al /data', - 'make-new-latest': f'cd /data && NEWDIR=$(date +%y%m%d) && mkdir -p {county_dirs} && ln -sfn $NEWDIR latest', + 'ls': 'ls -alR /data', } dl_roads = {}; dl_addrs = {}; dl_paths = {} @@ -70,7 +106,7 @@ def get_script_map(): cid = c['id'] name = c['name'] state = c['state'] - base = f'/data/latest/{cid}' + base = f'/data/{cid}' if c.get('roads_url'): dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip' @@ -123,19 +159,6 @@ def run_script(): if not script_name: return jsonify({'error': 'No script specified'}), 400 - # Check if /data/latest exists and is a symlink, not a regular directory - # Skip this check for make-new-latest, ls - skip_check_scripts = ['make-new-latest', 'ls'] - if script_name not in skip_check_scripts: - latest_path = '/data/latest' - if os.path.exists(latest_path) or os.path.islink(latest_path): - # Check if it's a directory but NOT a symlink - if os.path.isdir(latest_path) and not os.path.islink(latest_path): - return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400 - else: - # /data/latest doesn't exist at all - return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400 - script_map = get_script_map() if script_name not in script_map: @@ -145,10 +168,8 @@ def run_script(): # Handle both string commands and dict of county-specific commands if isinstance(script_config, str): - # Simple string command (like 'ls') cmd = ['bash', '-c', script_config] elif isinstance(script_config, dict): - # County-specific commands if not county: return jsonify({'error': 'County required for this script'}), 400 if county not in script_config: @@ -158,7 +179,7 @@ def run_script(): if isinstance(cmd_config, str): cmd = ['bash', '-c', cmd_config] else: - cmd = list(cmd_config) # Make a copy to avoid modifying the original + cmd = list(cmd_config) else: return jsonify({'error': 'Invalid script configuration'}), 400 @@ -179,7 +200,6 @@ def run_script(): running_processes[job_id] = process process_logs[job_id] = [] - # Stream output for line in process.stdout: process_logs[job_id].append(line) @@ -233,18 +253,11 @@ def cancel_job(): @app.route('/api/upload-file', methods=['POST']) def upload_file(): - """Upload a shapefile ZIP to a path under /data/latest""" + """Upload a shapefile ZIP to a path under /data""" dest = request.form.get('dest') - if not dest or not dest.startswith('/data/latest/'): + if not dest or not dest.startswith('/data/'): return jsonify({'error': 'Invalid destination path'}), 400 - latest = '/data/latest' - if os.path.exists(latest) or os.path.islink(latest): - if os.path.isdir(latest) and not os.path.islink(latest): - return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400 - else: - return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400 - if 'file' not in request.files: return jsonify({'error': 'No file provided'}), 400 @@ -271,7 +284,7 @@ def system_stats(): @app.route('/api/list-files') def list_files(): - """List available GeoJSON files""" + """List available GeoJSON files recursively with modification times.""" data_dir = '/data' files = { @@ -280,36 +293,63 @@ def list_files(): 'county': [] } - # Scan directories for geojson files if os.path.exists(data_dir): for root, dirs, filenames in os.walk(data_dir): + dirs[:] = [d for d in dirs if d != 'osm_cache'] for filename in filenames: if filename.endswith('.geojson'): - rel_path = os.path.relpath(os.path.join(root, filename), data_dir) + filepath = os.path.join(root, filename) + rel_path = os.path.relpath(filepath, data_dir).replace('\\', '/') + mtime = datetime.fromtimestamp(os.path.getmtime(filepath)).strftime('%Y-%m-%d %H:%M') + entry = {'path': rel_path, 'mtime': mtime} if 'diff' in filename.lower(): - files['diff'].append(rel_path) + files['diff'].append(entry) elif 'osm' in filename.lower(): - files['osm'].append(rel_path) - elif any(county in filename.lower() for county in ['lake', 'sumter']): - files['county'].append(rel_path) + files['osm'].append(entry) + else: + files['county'].append(entry) return jsonify(files) -@app.route('/api/latest-date') -def latest_date(): - """Return the folder name that /data/latest symlinks to (used as data date in filenames).""" - try: - target = os.path.basename(os.path.realpath('/data/latest')) - return jsonify({'date': target}) - except Exception: - return jsonify({'date': ''}) - @app.route('/data/') def serve_data(filename): - """Serve GeoJSON files""" + """Serve files from /data""" return send_from_directory('/data', filename) +# ── Config file download / upload ────────────────────────────────────────────── + +_ALLOWED_CONFIGS = {'counties.yml', 'exceptions.yml'} + +@app.route('/api/config/', methods=['GET']) +def download_config(filename): + if filename not in _ALLOWED_CONFIGS: + return jsonify({'error': 'Not found'}), 404 + return send_from_directory('/data', filename, as_attachment=True) + +@app.route('/api/config/', methods=['POST']) +def upload_config(filename): + if filename not in _ALLOWED_CONFIGS: + return jsonify({'error': 'Not found'}), 404 + if 'file' not in request.files: + return jsonify({'error': 'No file provided'}), 400 + f = request.files['file'] + try: + content = f.read().decode('utf-8') + data = yaml.safe_load(content) + if filename == 'counties.yml' and not isinstance((data or {}).get('counties'), list): + return jsonify({'error': 'Invalid counties.yml: missing counties list'}), 400 + if filename == 'exceptions.yml' and not isinstance((data or {}).get('corrections'), list): + return jsonify({'error': 'Invalid exceptions.yml: missing corrections list'}), 400 + except Exception as e: + return jsonify({'error': f'Invalid YAML: {e}'}), 400 + dest = f'/data/{filename}' + tmp = dest + '.tmp' + with open(tmp, 'w') as out: + out.write(content) + os.replace(tmp, dest) + return jsonify({'success': True}) + # ── Counties config ──────────────────────────────────────────────────────────── COUNTIES_FILE = '/data/counties.yml' @@ -423,14 +463,12 @@ def save_exceptions(): data = request.json corrections = data.get('corrections', []) - # Validate for item in corrections: if not isinstance(item.get('from'), str) or not item['from'].strip(): return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400 if not isinstance(item.get('to'), str) or not item['to'].strip(): return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400 - # Atomic write tmp = EXCEPTIONS_FILE + '.tmp' with open(tmp, 'w') as f: yaml.dump({'corrections': corrections}, f, diff --git a/web/templates/index.html b/web/templates/index.html index 965e483..4889dc7 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -65,6 +65,26 @@ .btn-topbar.purple:hover { background: #5a32a3; } + .btn-topbar-icon { + padding: 5px 8px; + border: none; + border-radius: 5px; + cursor: pointer; + font-size: 12px; + font-weight: 600; + background: #ffffff18; + color: #ddd; + text-decoration: none; + } + + .btn-topbar-icon:hover { background: #ffffff30; } + + .config-group { + display: flex; + align-items: center; + gap: 2px; + } + /* ── Main two-panel layout ── */ .main { flex: 1; @@ -355,9 +375,17 @@

OSM Import Tools

-
+
+ ↓ counties.yml + +
+
+ ↓ exceptions.yml + +
+
Counties Street Exceptions Open Map Viewer @@ -489,11 +517,11 @@
{% for county in counties %} {% if county.id in data_files %} - {% for file_name in data_files[county.id] %} + {% for file in data_files[county.id] %} {{ file_name }} + href="/data/{{ county.id }}/{{ file.name }}" + title="{{ county.id }}/{{ file.name }} — {{ file.mtime }}">{{ file.name }} {% endfor %} {% endif %} {% endfor %} @@ -568,7 +596,7 @@ // Step 1: upload or download const inputId = `upload-${county}-paths`; - const dest = `/data/latest/${county}/paths.shp.zip`; + const dest = `/data/${county}/paths.shp.zip`; if (pathsUrl === 'upload') { document.getElementById('paths-step-1-actions').innerHTML = `` + @@ -697,6 +725,25 @@ .finally(() => enableButtons()); } + function uploadConfig(filename, input) { + if (!input.files.length) return; + const formData = new FormData(); + formData.append('file', input.files[0]); + showStatus(`Uploading ${filename}...`, 'info'); + fetch(`/api/config/${filename}`, { method: 'POST', body: formData }) + .then(r => r.json()) + .then(data => { + if (data.error) { + showStatus(`Error: ${data.error}`, 'error'); + } else { + showStatus(`${filename} uploaded successfully.`, 'success'); + setTimeout(() => window.location.reload(), 800); + } + }) + .catch(err => showStatus(`Error: ${err.message}`, 'error')) + .finally(() => { input.value = ''; }); + } + function showStatus(msg, type) { const el = document.getElementById('status'); el.textContent = msg;