Flatten to /data/latest, add mtimes, add config download/upload

web/server.py

@@ -5,6 +5,8 @@ Flask web server for The Villages Import Tools
 from flask import Flask, render_template, jsonify, request, send_from_directory
 import subprocess
 import os
+import re
+import shutil
 import threading
 import json
 from datetime import datetime
@@ -21,24 +23,60 @@ app = Flask(__name__, static_folder='static', template_folder='templates')
 running_processes = {}
 process_logs = {}
 
-# Global var
+
-latest_path = "/data/latest"
+def _cleanup_legacy_structure():
+    """Remove dated folders and symlink from the old /data/YYMMDD layout,
+    migrating any files up to /data/{county}/ before deleting."""
+    data_dir = '/data'
+    if not os.path.exists(data_dir):
+        return
+
+    # Remove the legacy 'latest' symlink
+    symlink = os.path.join(data_dir, 'latest')
+    if os.path.islink(symlink):
+        os.unlink(symlink)
+    elif os.path.isdir(symlink):
+        # Safety: if it somehow became a real dir, leave it alone
+        pass
+
+    # Migrate and remove 6-digit dated folders (YYMMDD)
+    dated_pattern = re.compile(r'^\d{6}$')
+    for entry in os.listdir(data_dir):
+        entry_path = os.path.join(data_dir, entry)
+        if os.path.isdir(entry_path) and dated_pattern.match(entry):
+            for county_dir in os.listdir(entry_path):
+                src_county = os.path.join(entry_path, county_dir)
+                if os.path.isdir(src_county):
+                    dst_county = os.path.join(data_dir, county_dir)
+                    os.makedirs(dst_county, exist_ok=True)
+                    for fname in os.listdir(src_county):
+                        src_file = os.path.join(src_county, fname)
+                        dst_file = os.path.join(dst_county, fname)
+                        if os.path.isfile(src_file) and not os.path.exists(dst_file):
+                            shutil.move(src_file, dst_file)
+            shutil.rmtree(entry_path)
+
+_cleanup_legacy_structure()
+
 
 @app.route('/')
 def index():
     """Main index page with script execution buttons"""
     counties = get_counties()
 
-    # Get list of files
+    # Get list of files with modification times
     data_files = {}
-    if os.path.exists(latest_path):
+    for county in counties:
-        try:
+        county_dir = f'/data/{county["id"]}'
-            for countyFolder in os.listdir(latest_path):
+        if os.path.isdir(county_dir):
-                folder_path = '/data/latest/' + countyFolder
+            files = []
-                if os.path.isdir(folder_path):
+            for fname in sorted(os.listdir(county_dir)):
-                    data_files[countyFolder] = os.listdir(folder_path)
+                fpath = os.path.join(county_dir, fname)
-        except Exception:
+                if os.path.isfile(fpath):
-            pass
+                    mtime = datetime.fromtimestamp(os.path.getmtime(fpath)).strftime('%Y-%m-%d %H:%M')
+                    files.append({'name': fname, 'mtime': mtime})
+            if files:
+                data_files[county['id']] = files
 
     import json as _json
     return render_template('index.html',
@@ -55,10 +93,8 @@ def get_script_map():
     """Build script map dynamically from county configuration."""
     counties = get_counties()
 
-    county_dirs = ' '.join(f'$NEWDIR/{c["id"]}' for c in counties)
     sm = {
-        'ls': 'ls -al /data',
+        'ls': 'ls -alR /data',
-        'make-new-latest': f'cd /data && NEWDIR=$(date +%y%m%d) && mkdir -p {county_dirs} && ln -sfn $NEWDIR latest',
     }
 
     dl_roads = {}; dl_addrs = {}; dl_paths = {}
@@ -70,7 +106,7 @@ def get_script_map():
         cid   = c['id']
         name  = c['name']
         state = c['state']
-        base  = f'/data/latest/{cid}'
+        base  = f'/data/{cid}'
 
         if c.get('roads_url'):
             dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip'
@@ -123,19 +159,6 @@ def run_script():
     if not script_name:
         return jsonify({'error': 'No script specified'}), 400
 
-    # Check if /data/latest exists and is a symlink, not a regular directory
-    # Skip this check for make-new-latest, ls
-    skip_check_scripts = ['make-new-latest', 'ls']
-    if script_name not in skip_check_scripts:
-        latest_path = '/data/latest'
-        if os.path.exists(latest_path) or os.path.islink(latest_path):
-            # Check if it's a directory but NOT a symlink
-            if os.path.isdir(latest_path) and not os.path.islink(latest_path):
-                return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400
-        else:
-            # /data/latest doesn't exist at all
-            return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400
-
     script_map = get_script_map()
 
     if script_name not in script_map:
@@ -145,10 +168,8 @@ def run_script():
 
     # Handle both string commands and dict of county-specific commands
     if isinstance(script_config, str):
-        # Simple string command (like 'ls')
         cmd = ['bash', '-c', script_config]
     elif isinstance(script_config, dict):
-        # County-specific commands
         if not county:
             return jsonify({'error': 'County required for this script'}), 400
         if county not in script_config:
@@ -158,7 +179,7 @@ def run_script():
         if isinstance(cmd_config, str):
             cmd = ['bash', '-c', cmd_config]
         else:
-            cmd = list(cmd_config)  # Make a copy to avoid modifying the original
+            cmd = list(cmd_config)
 
     else:
         return jsonify({'error': 'Invalid script configuration'}), 400
@@ -179,7 +200,6 @@ def run_script():
             running_processes[job_id] = process
             process_logs[job_id] = []
 
-            # Stream output
             for line in process.stdout:
                 process_logs[job_id].append(line)
 
@@ -233,18 +253,11 @@ def cancel_job():
 
 @app.route('/api/upload-file', methods=['POST'])
 def upload_file():
-    """Upload a shapefile ZIP to a path under /data/latest"""
+    """Upload a shapefile ZIP to a path under /data"""
     dest = request.form.get('dest')
-    if not dest or not dest.startswith('/data/latest/'):
+    if not dest or not dest.startswith('/data/'):
         return jsonify({'error': 'Invalid destination path'}), 400
 
-    latest = '/data/latest'
-    if os.path.exists(latest) or os.path.islink(latest):
-        if os.path.isdir(latest) and not os.path.islink(latest):
-            return jsonify({'error': '/data/latest is a directory, not a symlink. Please run "Make New Latest" first.'}), 400
-    else:
-        return jsonify({'error': '/data/latest does not exist. Please run "Make New Latest" first.'}), 400
-
     if 'file' not in request.files:
         return jsonify({'error': 'No file provided'}), 400
 
@@ -271,7 +284,7 @@ def system_stats():
 
 @app.route('/api/list-files')
 def list_files():
-    """List available GeoJSON files"""
+    """List available GeoJSON files recursively with modification times."""
     data_dir = '/data'
 
     files = {
@@ -280,36 +293,63 @@ def list_files():
         'county': []
     }
 
-    # Scan directories for geojson files
     if os.path.exists(data_dir):
         for root, dirs, filenames in os.walk(data_dir):
+            dirs[:] = [d for d in dirs if d != 'osm_cache']
             for filename in filenames:
                 if filename.endswith('.geojson'):
-                    rel_path = os.path.relpath(os.path.join(root, filename), data_dir)
+                    filepath = os.path.join(root, filename)
+                    rel_path = os.path.relpath(filepath, data_dir).replace('\\', '/')
+                    mtime = datetime.fromtimestamp(os.path.getmtime(filepath)).strftime('%Y-%m-%d %H:%M')
+                    entry = {'path': rel_path, 'mtime': mtime}
 
                     if 'diff' in filename.lower():
-                        files['diff'].append(rel_path)
+                        files['diff'].append(entry)
                     elif 'osm' in filename.lower():
-                        files['osm'].append(rel_path)
+                        files['osm'].append(entry)
-                    elif any(county in filename.lower() for county in ['lake', 'sumter']):
+                    else:
-                        files['county'].append(rel_path)
+                        files['county'].append(entry)
 
     return jsonify(files)
 
-@app.route('/api/latest-date')
-def latest_date():
-    """Return the folder name that /data/latest symlinks to (used as data date in filenames)."""
-    try:
-        target = os.path.basename(os.path.realpath('/data/latest'))
-        return jsonify({'date': target})
-    except Exception:
-        return jsonify({'date': ''})
-
 @app.route('/data/<path:filename>')
 def serve_data(filename):
-    """Serve GeoJSON files"""
+    """Serve files from /data"""
     return send_from_directory('/data', filename)
 
+# ── Config file download / upload ──────────────────────────────────────────────
+
+_ALLOWED_CONFIGS = {'counties.yml', 'exceptions.yml'}
+
+@app.route('/api/config/<filename>', methods=['GET'])
+def download_config(filename):
+    if filename not in _ALLOWED_CONFIGS:
+        return jsonify({'error': 'Not found'}), 404
+    return send_from_directory('/data', filename, as_attachment=True)
+
+@app.route('/api/config/<filename>', methods=['POST'])
+def upload_config(filename):
+    if filename not in _ALLOWED_CONFIGS:
+        return jsonify({'error': 'Not found'}), 404
+    if 'file' not in request.files:
+        return jsonify({'error': 'No file provided'}), 400
+    f = request.files['file']
+    try:
+        content = f.read().decode('utf-8')
+        data = yaml.safe_load(content)
+        if filename == 'counties.yml' and not isinstance((data or {}).get('counties'), list):
+            return jsonify({'error': 'Invalid counties.yml: missing counties list'}), 400
+        if filename == 'exceptions.yml' and not isinstance((data or {}).get('corrections'), list):
+            return jsonify({'error': 'Invalid exceptions.yml: missing corrections list'}), 400
+    except Exception as e:
+        return jsonify({'error': f'Invalid YAML: {e}'}), 400
+    dest = f'/data/{filename}'
+    tmp = dest + '.tmp'
+    with open(tmp, 'w') as out:
+        out.write(content)
+    os.replace(tmp, dest)
+    return jsonify({'success': True})
+
 # ── Counties config ────────────────────────────────────────────────────────────
 
 COUNTIES_FILE = '/data/counties.yml'
@@ -423,14 +463,12 @@ def save_exceptions():
     data = request.json
     corrections = data.get('corrections', [])
 
-    # Validate
     for item in corrections:
         if not isinstance(item.get('from'), str) or not item['from'].strip():
             return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400
         if not isinstance(item.get('to'), str) or not item['to'].strip():
             return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400
 
-    # Atomic write
     tmp = EXCEPTIONS_FILE + '.tmp'
     with open(tmp, 'w') as f:
         yaml.dump({'corrections': corrections}, f,

web/templates/index.html

@@ -65,6 +65,26 @@
 
         .btn-topbar.purple:hover { background: #5a32a3; }
 
+        .btn-topbar-icon {
+            padding: 5px 8px;
+            border: none;
+            border-radius: 5px;
+            cursor: pointer;
+            font-size: 12px;
+            font-weight: 600;
+            background: #ffffff18;
+            color: #ddd;
+            text-decoration: none;
+        }
+
+        .btn-topbar-icon:hover { background: #ffffff30; }
+
+        .config-group {
+            display: flex;
+            align-items: center;
+            gap: 2px;
+        }
+
         /* ── Main two-panel layout ── */
         .main {
             flex: 1;
@@ -355,9 +375,17 @@
     <div class="topbar">
         <h1>OSM Import Tools</h1>
         <div class="topbar-sep"></div>
-        <button class="btn-topbar" onclick="runScript('make-new-latest', '')">New Latest</button>
         <button class="btn-topbar" onclick="runScript('ls', '')">List Files</button>
         <div style="flex:1"></div>
+        <div class="config-group">
+            <a href="/api/config/counties.yml" class="btn-topbar-icon" title="Download counties.yml">↓ counties.yml</a>
+            <label class="btn-topbar-icon" title="Upload counties.yml" style="cursor:pointer">↑<input type="file" style="display:none" accept=".yml,.yaml" onchange="uploadConfig('counties.yml', this)"></label>
+        </div>
+        <div class="config-group">
+            <a href="/api/config/exceptions.yml" class="btn-topbar-icon" title="Download exceptions.yml">↓ exceptions.yml</a>
+            <label class="btn-topbar-icon" title="Upload exceptions.yml" style="cursor:pointer">↑<input type="file" style="display:none" accept=".yml,.yaml" onchange="uploadConfig('exceptions.yml', this)"></label>
+        </div>
+        <div class="topbar-sep"></div>
         <a href="/counties" class="btn-topbar">Counties</a>
         <a href="/exceptions" class="btn-topbar">Street Exceptions</a>
         <a href="/map" class="btn-topbar purple">Open Map Viewer</a>
@@ -489,11 +517,11 @@
                 <div class="file-links">
                     {% for county in counties %}
                         {% if county.id in data_files %}
-                        {% for file_name in data_files[county.id] %}
+                        {% for file in data_files[county.id] %}
                         <a class="file-link"
                            style="--link-color: {{ county.color }}"
-                           href="/data/latest/{{ county.id }}/{{ file_name }}"
+                           href="/data/{{ county.id }}/{{ file.name }}"
-                           title="{{ county.id }}/{{ file_name }}">{{ file_name }}</a>
+                           title="{{ county.id }}/{{ file.name }} — {{ file.mtime }}">{{ file.name }}</a>
                         {% endfor %}
                         {% endif %}
                     {% endfor %}
@@ -568,7 +596,7 @@
 
         // Step 1: upload or download
         const inputId = `upload-${county}-paths`;
-        const dest = `/data/latest/${county}/paths.shp.zip`;
+        const dest = `/data/${county}/paths.shp.zip`;
         if (pathsUrl === 'upload') {
             document.getElementById('paths-step-1-actions').innerHTML =
                 `<input type="file" id="${inputId}" class="upload-input" accept=".zip">` +
@@ -697,6 +725,25 @@
         .finally(() => enableButtons());
     }
 
+    function uploadConfig(filename, input) {
+        if (!input.files.length) return;
+        const formData = new FormData();
+        formData.append('file', input.files[0]);
+        showStatus(`Uploading ${filename}...`, 'info');
+        fetch(`/api/config/${filename}`, { method: 'POST', body: formData })
+        .then(r => r.json())
+        .then(data => {
+            if (data.error) {
+                showStatus(`Error: ${data.error}`, 'error');
+            } else {
+                showStatus(`${filename} uploaded successfully.`, 'success');
+                setTimeout(() => window.location.reload(), 800);
+            }
+        })
+        .catch(err => showStatus(`Error: ${err.message}`, 'error'))
+        .finally(() => { input.value = ''; });
+    }
+
     function showStatus(msg, type) {
         const el = document.getElementById('status');
         el.textContent = msg;