will/osm-import-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add auth system, fix map defaults, controls overflow, fix latest-date ref

Browse Source
This commit is contained in:
zyphlar
2026-04-22 17:03:15 -07:00
parent 6880382011
commit 24c27767ec
8 changed files with 288 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -2,4 +2,5 @@ __pycache__
desktop.ini
*.geojson
osm_cache/
.claude
.claude
stack.env
docker-compose.yml
+2
View File
@@ -9,6 +9,8 @@ services:
- "5000:5000"
volumes:
- ./data:/data
env_file:
- stack.env
environment:
- FLASK_ENV=development
restart: unless-stopped
stack.env.example
+12
View File
@@ -0,0 +1,12 @@
# OSM Import Tools – environment configuration
# Edit these values before deploying. In Portainer GitOps you can also
# override them in the stack's Environment Variables UI.
# Password for the built-in 'admin' account.
# The container seeds this on every startup, so changing it here
# and redeploying will update the password automatically.
ADMIN_PASSWORD=changeme
# Secret key for Flask session signing.
# Generate a strong random value, e.g.: python -c "import secrets; print(secrets.token_hex(32))"
FLASK_SECRET_KEY=change-this-to-a-long-random-string
web/server.py
+133 -46
View File
@@ -2,10 +2,13 @@
"""
Flask web server for The Villages Import Tools
"""
from flask import Flask, render_template, jsonify, request, send_from_directory
from functools import wraps
from flask import Flask, render_template, jsonify, request, send_from_directory, session, redirect, url_for
from werkzeug.security import generate_password_hash, check_password_hash
import subprocess
import os
import re
import secrets
import shutil
import threading
import json
@@ -19,7 +22,105 @@ except ImportError:
app = Flask(__name__, static_folder='static', template_folder='templates')
# Store running processes
# ── Secret key ────────────────────────────────────────────────────────────────
def _get_or_create_secret_key():
key = os.environ.get('FLASK_SECRET_KEY', '')
if key:
return key
key_file = '/data/.secret_key'
if os.path.exists(key_file):
with open(key_file) as f:
return f.read().strip()
key = secrets.token_hex(32)
try:
os.makedirs('/data', exist_ok=True)
with open(key_file, 'w') as f:
f.write(key)
except Exception:
pass
return key
app.secret_key = _get_or_create_secret_key()
# ── Users / auth ───────────────────────────────────────────────────────────────
USERS_FILE = '/data/users.yml'
def _load_users():
try:
with open(USERS_FILE) as f:
data = yaml.safe_load(f) or {}
return {u['username']: u['password_hash'] for u in data.get('users', [])}
except Exception:
return {}
def _save_users(users_dict):
users_list = [{'username': u, 'password_hash': h} for u, h in sorted(users_dict.items())]
tmp = USERS_FILE + '.tmp'
with open(tmp, 'w') as f:
yaml.dump({'users': users_list}, f, default_flow_style=False, allow_unicode=True)
os.replace(tmp, USERS_FILE)
def _seed_admin():
"""Create or update the admin user from the ADMIN_PASSWORD env var."""
password = os.environ.get('ADMIN_PASSWORD', '')
if not password:
return
users = _load_users()
users['admin'] = generate_password_hash(password)
_save_users(users)
_seed_admin()
def require_auth(f):
@wraps(f)
def decorated(*args, **kwargs):
if not session.get('username'):
if request.is_json or request.path.startswith('/api/'):
return jsonify({'error': 'Authentication required'}), 401
return redirect(url_for('login_page', next=request.path))
return f(*args, **kwargs)
return decorated
@app.route('/login', methods=['GET'])
def login_page():
if session.get('username'):
return redirect(request.args.get('next', '/'))
return render_template('login.html', next=request.args.get('next', '/'), error=None)
@app.route('/login', methods=['POST'])
def login():
username = request.form.get('username', '').strip()
password = request.form.get('password', '')
next_url = request.form.get('next', '/')
users = _load_users()
if username and username in users and check_password_hash(users[username], password):
session['username'] = username
return redirect(next_url)
return render_template('login.html', next=next_url, error='Invalid username or password')
@app.route('/logout', methods=['POST'])
def logout():
session.clear()
return redirect('/')
@app.route('/api/auth-status')
def auth_status():
username = session.get('username')
return jsonify({'authenticated': bool(username), 'username': username or ''})
# ── Store running processes ────────────────────────────────────────────────────
running_processes = {}
process_logs = {}
@@ -31,15 +132,10 @@ def _cleanup_legacy_structure():
if not os.path.exists(data_dir):
return
# Remove the legacy 'latest' symlink
symlink = os.path.join(data_dir, 'latest')
if os.path.islink(symlink):
os.unlink(symlink)
elif os.path.isdir(symlink):
# Safety: if it somehow became a real dir, leave it alone
pass
# Migrate and remove 6-digit dated folders (YYMMDD)
dated_pattern = re.compile(r'^\d{6}$')
for entry in os.listdir(data_dir):
entry_path = os.path.join(data_dir, entry)
@@ -64,7 +160,6 @@ def index():
"""Main index page with script execution buttons"""
counties = get_counties()
# Get list of files with modification times
data_files = {}
for county in counties:
county_dir = f'/data/{county["id"]}'
@@ -82,7 +177,8 @@ def index():
return render_template('index.html',
counties=counties,
counties_json=_json.dumps(counties),
data_files=data_files)
data_files=data_files,
current_user=session.get('username'))
@app.route('/map')
def map_viewer():
@@ -149,6 +245,7 @@ def get_script_map():
return sm
@app.route('/api/run-script', methods=['POST'])
@require_auth
def run_script():
"""Execute a script in the background"""
data = request.json
@@ -166,7 +263,6 @@ def run_script():
script_config = script_map[script_name]
# Handle both string commands and dict of county-specific commands
if isinstance(script_config, str):
cmd = ['bash', '-c', script_config]
elif isinstance(script_config, dict):
@@ -184,10 +280,8 @@ def run_script():
else:
return jsonify({'error': 'Invalid script configuration'}), 400
# Generate a unique job ID
job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
# Start process in background
def run_command():
try:
process = subprocess.Popen(
@@ -223,14 +317,10 @@ def job_status(job_id):
"""Get status and logs for a job"""
is_running = job_id in running_processes
logs = process_logs.get(job_id, [])
return jsonify({
'job_id': job_id,
'running': is_running,
'logs': logs
})
return jsonify({'job_id': job_id, 'running': is_running, 'logs': logs})
@app.route('/api/cancel-job', methods=['POST'])
@require_auth
def cancel_job():
"""Cancel a running job"""
data = request.json
@@ -238,7 +328,6 @@ def cancel_job():
if not job_id:
return jsonify({'error': 'No job ID specified'}), 400
if job_id not in running_processes:
return jsonify({'error': 'Job not found or already completed'}), 404
@@ -251,7 +340,27 @@ def cancel_job():
except Exception as e:
return jsonify({'error': str(e)}), 500
@app.route('/api/upload-file', methods=['POST'])
@require_auth
def upload_file():
"""Upload a shapefile ZIP to a path under /data"""
dest = request.form.get('dest')
if not dest or not dest.startswith('/data/'):
return jsonify({'error': 'Invalid destination path'}), 400
if 'file' not in request.files:
return jsonify({'error': 'No file provided'}), 400
f = request.files['file']
if not f.filename.lower().endswith('.zip'):
return jsonify({'error': 'File must be a ZIP archive (.zip)'}), 400
os.makedirs(os.path.dirname(dest), exist_ok=True)
f.save(dest)
return jsonify({'success': True, 'path': dest})
@app.route('/api/clear-data', methods=['POST'])
@require_auth
def clear_data():
"""Delete all files from every county data directory under /data."""
counties = get_counties()
@@ -273,24 +382,6 @@ def clear_data():
return jsonify({'error': '; '.join(errors)}), 500
return jsonify({'success': True, 'removed': removed})
@app.route('/api/upload-file', methods=['POST'])
def upload_file():
"""Upload a shapefile ZIP to a path under /data"""
dest = request.form.get('dest')
if not dest or not dest.startswith('/data/'):
return jsonify({'error': 'Invalid destination path'}), 400
if 'file' not in request.files:
return jsonify({'error': 'No file provided'}), 400
f = request.files['file']
if not f.filename.lower().endswith('.zip'):
return jsonify({'error': 'File must be a ZIP archive (.zip)'}), 400
os.makedirs(os.path.dirname(dest), exist_ok=True)
f.save(dest)
return jsonify({'success': True, 'path': dest})
@app.route('/api/system-stats')
def system_stats():
"""Return current CPU, memory, and disk usage percentages."""
@@ -308,12 +399,7 @@ def system_stats():
def list_files():
"""List available GeoJSON files recursively with modification times."""
data_dir = '/data'
files = {
'diff': [],
'osm': [],
'county': []
}
files = {'diff': [], 'osm': [], 'county': []}
if os.path.exists(data_dir):
for root, dirs, filenames in os.walk(data_dir):
@@ -324,7 +410,6 @@ def list_files():
rel_path = os.path.relpath(filepath, data_dir).replace('\\', '/')
mtime = datetime.fromtimestamp(os.path.getmtime(filepath)).strftime('%Y-%m-%d %H:%M')
entry = {'path': rel_path, 'mtime': mtime}
if 'diff' in filename.lower():
files['diff'].append(entry)
elif 'osm' in filename.lower():
@@ -344,12 +429,14 @@ def serve_data(filename):
_ALLOWED_CONFIGS = {'counties.yml', 'exceptions.yml'}
@app.route('/api/config/<filename>', methods=['GET'])
@require_auth
def download_config(filename):
if filename not in _ALLOWED_CONFIGS:
return jsonify({'error': 'Not found'}), 404
return send_from_directory('/data', filename, as_attachment=True)
@app.route('/api/config/<filename>', methods=['POST'])
@require_auth
def upload_config(filename):
if filename not in _ALLOWED_CONFIGS:
return jsonify({'error': 'Not found'}), 404
@@ -426,6 +513,7 @@ def get_counties_api():
return jsonify({'error': str(e)}), 500
@app.route('/api/counties', methods=['POST'])
@require_auth
def save_counties():
data = request.json
counties = data.get('counties', [])
@@ -481,16 +569,15 @@ def get_exceptions():
@app.route('/api/exceptions', methods=['POST'])
@require_auth
def save_exceptions():
data = request.json
corrections = data.get('corrections', [])
for item in corrections:
if not isinstance(item.get('from'), str) or not item['from'].strip():
return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400
if not isinstance(item.get('to'), str) or not item['to'].strip():
return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400
tmp = EXCEPTIONS_FILE + '.tmp'
with open(tmp, 'w') as f:
yaml.dump({'corrections': corrections}, f,
web/static/map.js
+1 -6
View File
@@ -1059,12 +1059,7 @@ async function saveAcceptedItems() {
}
});
// Get the date of the loaded data from the server
let dataDate = '';
try {
const dateResp = await fetch('/api/latest-date');
if (dateResp.ok) dataDate = (await dateResp.json()).date || '';
} catch (_) {}
const dataDate = new Date().toISOString().slice(0, 10).replace(/-/g, '');
const prefix = [loadedCounty, loadedDataType, dataDate].filter(Boolean).join('-');
function triggerDownload(features, label) {
web/templates/index.html
+21
View File
@@ -378,6 +378,7 @@
<button class="btn-topbar" onclick="runScript('ls', '')">List Files</button>
<button class="btn-topbar" onclick="clearData()" style="color:#f88;">Clear Data</button>
<div style="flex:1"></div>
{% if current_user %}
<div class="config-group">
<a href="/api/config/counties.yml" class="btn-topbar-icon" title="Download counties.yml">↓ counties.yml</a>
<label class="btn-topbar-icon" title="Upload counties.yml" style="cursor:pointer">↑<input type="file" style="display:none" accept=".yml,.yaml" onchange="uploadConfig('counties.yml', this)"></label>
@@ -387,9 +388,19 @@
<label class="btn-topbar-icon" title="Upload exceptions.yml" style="cursor:pointer">↑<input type="file" style="display:none" accept=".yml,.yaml" onchange="uploadConfig('exceptions.yml', this)"></label>
</div>
<div class="topbar-sep"></div>
{% endif %}
<a href="/counties" class="btn-topbar">Counties</a>
<a href="/exceptions" class="btn-topbar">Street Exceptions</a>
<a href="/map" class="btn-topbar purple">Open Map Viewer</a>
<div class="topbar-sep"></div>
{% if current_user %}
<span style="font-size:11px;color:#888;">{{ current_user }}</span>
<form method="POST" action="/logout" style="display:inline;margin:0">
<button type="submit" class="btn-topbar">Logout</button>
</form>
{% else %}
<a href="/login" class="btn-topbar" style="color:#f0a830;">Login</a>
{% endif %}
</div>
<div class="main">
@@ -564,10 +575,17 @@
<script>
const COUNTY_CONFIG = {{ counties_json | safe }};
const IS_AUTHENTICATED = {{ 'true' if current_user else 'false' }};
let activeCounty = COUNTY_CONFIG.length ? COUNTY_CONFIG[0].id : '';
let currentJobId = null;
let logCheckInterval = null;
function requireLogin() {
showStatus('Not authenticated – please log in.', 'error');
setTimeout(() => { window.location.href = '/login?next=' + encodeURIComponent(window.location.pathname); }, 1200);
return false;
}
function selectCounty(county) {
activeCounty = county;
const config = COUNTY_CONFIG.find(c => c.id === county);
@@ -619,6 +637,7 @@
/* ── Script runner ── */
function runScript(scriptName) {
if (!IS_AUTHENTICATED) return requireLogin();
const logsEl = document.getElementById('logs');
logsEl.textContent = '';
showStatus(`Running: ${scriptName} (${activeCounty})`, 'info');
@@ -699,6 +718,7 @@
}
function uploadFile(inputId, dest) {
if (!IS_AUTHENTICATED) return requireLogin();
const input = document.getElementById(inputId);
if (!input || !input.files.length) {
showStatus('Please select a file first.', 'error');
@@ -727,6 +747,7 @@
}
function clearData() {
if (!IS_AUTHENTICATED) return requireLogin();
if (!confirm('Delete all downloaded and processed files from all county folders?')) return;
fetch('/api/clear-data', { method: 'POST' })
.then(r => r.json())
web/templates/login.html
+113
View File
@@ -0,0 +1,113 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Login – OSM Import Tools</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
height: 100vh;
display: flex;
align-items: center;
justify-content: center;
background: #0f0f1a;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
}
.login-box {
background: #1a1a2e;
border-radius: 10px;
padding: 36px 32px;
width: 320px;
box-shadow: 0 8px 32px rgba(0,0,0,0.4);
}
.login-box h1 {
font-size: 18px;
font-weight: 700;
color: #fff;
margin-bottom: 6px;
}
.login-box .subtitle {
font-size: 12px;
color: #666;
margin-bottom: 28px;
}
.error-msg {
background: #3a1e1e;
color: #e07070;
border-radius: 5px;
padding: 8px 12px;
font-size: 13px;
margin-bottom: 16px;
}
label {
display: block;
font-size: 11px;
font-weight: 600;
color: #888;
text-transform: uppercase;
letter-spacing: 0.5px;
margin-bottom: 5px;
}
input[type="text"],
input[type="password"] {
width: 100%;
padding: 9px 12px;
border: 1px solid #333;
border-radius: 5px;
background: #0f0f1a;
color: #ddd;
font-size: 14px;
margin-bottom: 16px;
outline: none;
transition: border-color 0.15s;
}
input[type="text"]:focus,
input[type="password"]:focus {
border-color: #6f42c1;
}
button[type="submit"] {
width: 100%;
padding: 10px;
background: #6f42c1;
color: white;
border: none;
border-radius: 5px;
font-size: 14px;
font-weight: 600;
cursor: pointer;
transition: background 0.15s;
}
button[type="submit"]:hover { background: #5a32a3; }
</style>
</head>
<body>
<div class="login-box">
<h1>OSM Import Tools</h1>
<div class="subtitle">Sign in to continue</div>
{% if error %}
<div class="error-msg">{{ error }}</div>
{% endif %}
<form method="POST" action="/login">
<input type="hidden" name="next" value="{{ next }}">
<label for="username">Username</label>
<input type="text" id="username" name="username" autofocus autocomplete="username" required>
<label for="password">Password</label>
<input type="password" id="password" name="password" autocomplete="current-password" required>
<button type="submit">Log in</button>
</form>
</div>
</body>
</html>
web/templates/map.html
+4 -2
View File
@@ -34,6 +34,8 @@
border-radius: 8px;
box-shadow: 0 2px 10px rgba(0,0,0,0.2);
min-width: 200px;
max-height: calc(100vh - 20px);
overflow-y: auto;
}
.controls h3 {
@@ -291,7 +293,7 @@
<span>Diff Layer</span>
</div>
<div class="layer-item" draggable="true" data-layer="osm">
<input type="checkbox" id="osmToggle" checked>
<input type="checkbox" id="osmToggle">
<span>OSM Layer (Gray)</span>
</div>
<div class="layer-item" draggable="true" data-layer="county">
@@ -306,7 +308,7 @@
Show Added (Green)
</label>
<label>
<input type="checkbox" id="showRemoved" checked>
<input type="checkbox" id="showRemoved">
Show Removed (Red)
</label>
<label>