will/osm-import-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Flatten to /data/latest, add mtimes, add config download/upload

Browse Source
This commit is contained in:
zyphlar
2026-04-21 22:49:45 -07:00
parent 7d6fc1c62e
commit 7aaff6be35
2 changed files with 150 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/server.py
+98 -60
View File
@@ -5,6 +5,8 @@ Flask web server for The Villages Import Tools
from flask import Flask, render_template, jsonify, request, send_from_directory
import subprocess
import os
import re
import shutil
import threading
import json
from datetime import datetime
@@ -21,24 +23,60 @@ app = Flask(__name__, static_folder='static', template_folder='templates')
running_processes = {}
process_logs = {}
# Global var
latest_path = "/data/latest"
def _cleanup_legacy_structure():
"""Remove dated folders and symlink from the old /data/YYMMDD layout,
migrating any files up to /data/{county}/ before deleting."""
data_dir = '/data'
if not os.path.exists(data_dir):
return
# Remove the legacy 'latest' symlink
symlink = os.path.join(data_dir, 'latest')
if os.path.islink(symlink):
os.unlink(symlink)
elif os.path.isdir(symlink):
# Safety: if it somehow became a real dir, leave it alone
pass
# Migrate and remove 6-digit dated folders (YYMMDD)
dated_pattern = re.compile(r'^\d{6}$')
for entry in os.listdir(data_dir):
entry_path = os.path.join(data_dir, entry)
if os.path.isdir(entry_path) and dated_pattern.match(entry):
for county_dir in os.listdir(entry_path):
src_county = os.path.join(entry_path, county_dir)
if os.path.isdir(src_county):
dst_county = os.path.join(data_dir, county_dir)
os.makedirs(dst_county, exist_ok=True)
for fname in os.listdir(src_county):
src_file = os.path.join(src_county, fname)
dst_file = os.path.join(dst_county, fname)
if os.path.isfile(src_file) and not os.path.exists(dst_file):
shutil.move(src_file, dst_file)
shutil.rmtree(entry_path)
_cleanup_legacy_structure()
@app.route('/')
def index():
"""Main index page with script execution buttons"""
counties = get_counties()
# Get list of files
# Get list of files with modification times
data_files = {}
if os.path.exists(latest_path):
try:
for countyFolder in os.listdir(latest_path):
folder_path = '/data/latest/' + countyFolder
if os.path.isdir(folder_path):
data_files[countyFolder] = os.listdir(folder_path)
except Exception:
pass
for county in counties:
county_dir = f'/data/{county["id"]}'
if os.path.isdir(county_dir):
files = []
for fname in sorted(os.listdir(county_dir)):
fpath = os.path.join(county_dir, fname)
if os.path.isfile(fpath):
mtime = datetime.fromtimestamp(os.path.getmtime(fpath)).strftime('%Y-%m-%d %H:%M')
files.append({'name': fname, 'mtime': mtime})
if files:
data_files[county['id']] = files
import json as _json
return render_template('index.html',
@@ -55,10 +93,8 @@ def get_script_map():
"""Build script map dynamically from county configuration."""
counties = get_counties()
county_dirs = ' '.join(f'$NEWDIR/{c["id"]}' for c in counties)
sm = {
'ls': 'ls -al /data',
'make-new-latest': f'cd /data && NEWDIR=$(date +%y%m%d) && mkdir -p {county_dirs} && ln -sfn $NEWDIR latest',
'ls': 'ls -alR /data',
}
dl_roads = {}; dl_addrs = {}; dl_paths = {}
@@ -70,7 +106,7 @@ def get_script_map():
cid = c['id']
name = c['name']
state = c['state']
base = f'/data/latest/{cid}'
base = f'/data/{cid}'
if c.get('roads_url'):
dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip'
@@ -123,19 +159,6 @@ def run_script():
if not script_name:
return jsonify({'error': 'No script specified'}), 400
# Check if /data/latest exists and is a symlink, not a regular directory
# Skip this check for make-new-latest, ls
skip_check_scripts = ['make-new-latest', 'ls']
if script_name not in skip_check_scripts:
latest_path = '/data/latest'
if os.path.exists(latest_path) or os.path.islink(latest_path):
# Check if it's a directory but NOT a symlink
if os.path.isdir(latest_path) and not os.path.islink(latest_path):
return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400
else:
# /data/latest doesn't exist at all
return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400
script_map = get_script_map()
if script_name not in script_map:
@@ -145,10 +168,8 @@ def run_script():
# Handle both string commands and dict of county-specific commands
if isinstance(script_config, str):
# Simple string command (like 'ls')
cmd = ['bash', '-c', script_config]
elif isinstance(script_config, dict):
# County-specific commands
if not county:
return jsonify({'error': 'County required for this script'}), 400
if county not in script_config:
@@ -158,7 +179,7 @@ def run_script():
if isinstance(cmd_config, str):
cmd = ['bash', '-c', cmd_config]
else:
cmd = list(cmd_config) # Make a copy to avoid modifying the original
cmd = list(cmd_config)
else:
return jsonify({'error': 'Invalid script configuration'}), 400
@@ -179,7 +200,6 @@ def run_script():
running_processes[job_id] = process
process_logs[job_id] = []
# Stream output
for line in process.stdout:
process_logs[job_id].append(line)
@@ -233,18 +253,11 @@ def cancel_job():
@app.route('/api/upload-file', methods=['POST'])
def upload_file():
"""Upload a shapefile ZIP to a path under /data/latest"""
"""Upload a shapefile ZIP to a path under /data"""
dest = request.form.get('dest')
if not dest or not dest.startswith('/data/latest/'):
if not dest or not dest.startswith('/data/'):
return jsonify({'error': 'Invalid destination path'}), 400
latest = '/data/latest'
if os.path.exists(latest) or os.path.islink(latest):
if os.path.isdir(latest) and not os.path.islink(latest):
return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400
else:
return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400
if 'file' not in request.files:
return jsonify({'error': 'No file provided'}), 400
@@ -271,7 +284,7 @@ def system_stats():
@app.route('/api/list-files')
def list_files():
"""List available GeoJSON files"""
"""List available GeoJSON files recursively with modification times."""
data_dir = '/data'
files = {
@@ -280,36 +293,63 @@ def list_files():
'county': []
}
# Scan directories for geojson files
if os.path.exists(data_dir):
for root, dirs, filenames in os.walk(data_dir):
dirs[:] = [d for d in dirs if d != 'osm_cache']
for filename in filenames:
if filename.endswith('.geojson'):
rel_path = os.path.relpath(os.path.join(root, filename), data_dir)
filepath = os.path.join(root, filename)
rel_path = os.path.relpath(filepath, data_dir).replace('\\', '/')
mtime = datetime.fromtimestamp(os.path.getmtime(filepath)).strftime('%Y-%m-%d %H:%M')
entry = {'path': rel_path, 'mtime': mtime}
if 'diff' in filename.lower():
files['diff'].append(rel_path)
files['diff'].append(entry)
elif 'osm' in filename.lower():
files['osm'].append(rel_path)
elif any(county in filename.lower() for county in ['lake', 'sumter']):
files['county'].append(rel_path)
files['osm'].append(entry)
else:
files['county'].append(entry)
return jsonify(files)
@app.route('/api/latest-date')
def latest_date():
"""Return the folder name that /data/latest symlinks to (used as data date in filenames)."""
try:
target = os.path.basename(os.path.realpath('/data/latest'))
return jsonify({'date': target})
except Exception:
return jsonify({'date': ''})
@app.route('/data/<path:filename>')
def serve_data(filename):
"""Serve GeoJSON files"""
"""Serve files from /data"""
return send_from_directory('/data', filename)
# ── Config file download / upload ──────────────────────────────────────────────
_ALLOWED_CONFIGS = {'counties.yml', 'exceptions.yml'}
@app.route('/api/config/<filename>', methods=['GET'])
def download_config(filename):
if filename not in _ALLOWED_CONFIGS:
return jsonify({'error': 'Not found'}), 404
return send_from_directory('/data', filename, as_attachment=True)
@app.route('/api/config/<filename>', methods=['POST'])
def upload_config(filename):
if filename not in _ALLOWED_CONFIGS:
return jsonify({'error': 'Not found'}), 404
if 'file' not in request.files:
return jsonify({'error': 'No file provided'}), 400
f = request.files['file']
try:
content = f.read().decode('utf-8')
data = yaml.safe_load(content)
if filename == 'counties.yml' and not isinstance((data or {}).get('counties'), list):
return jsonify({'error': 'Invalid counties.yml: missing counties list'}), 400
if filename == 'exceptions.yml' and not isinstance((data or {}).get('corrections'), list):
return jsonify({'error': 'Invalid exceptions.yml: missing corrections list'}), 400
except Exception as e:
return jsonify({'error': f'Invalid YAML: {e}'}), 400
dest = f'/data/{filename}'
tmp = dest + '.tmp'
with open(tmp, 'w') as out:
out.write(content)
os.replace(tmp, dest)
return jsonify({'success': True})
# ── Counties config ────────────────────────────────────────────────────────────
COUNTIES_FILE = '/data/counties.yml'
@@ -423,14 +463,12 @@ def save_exceptions():
data = request.json
corrections = data.get('corrections', [])
# Validate
for item in corrections:
if not isinstance(item.get('from'), str) or not item['from'].strip():
return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400
if not isinstance(item.get('to'), str) or not item['to'].strip():
return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400
# Atomic write
tmp = EXCEPTIONS_FILE + '.tmp'
with open(tmp, 'w') as f:
yaml.dump({'corrections': corrections}, f,