From 2254d293a0963273ab222d6dcb1ae4a6851b7ee2 Mon Sep 17 00:00:00 2001 From: Will Bradley Date: Thu, 4 Dec 2025 05:23:33 -0800 Subject: [PATCH] Get web server going --- .dockerignore | 21 ++ DOCKER.md | 117 +++++++ Dockerfile | 33 ++ README.md | 3 +- compare-addresses.py | 18 +- diff-highways.py | 8 +- docker-compose.yml | 14 + requirements.txt | 7 + web/app.js | 270 ++++++++++++++-- web/index.html | 57 +++- web/server.py | 215 ++++++++++++ web/static/map.js | 683 +++++++++++++++++++++++++++++++++++++++ web/templates/index.html | 311 ++++++++++++++++++ web/templates/map.html | 207 ++++++++++++ 14 files changed, 1916 insertions(+), 48 deletions(-) create mode 100644 .dockerignore create mode 100644 DOCKER.md create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 web/server.py create mode 100644 web/static/map.js create mode 100644 web/templates/index.html create mode 100644 web/templates/map.html diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c3a59c4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +*.so +*.egg +*.egg-info +dist +build +.git +.gitignore +.vscode +.idea +*.swp +*.swo +*~ +.DS_Store +venv/ +env/ +.env diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 0000000..2b69cd7 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,117 @@ +# Docker Deployment Guide + +## Quick Start + +### Using Docker Compose (Recommended) + +1. Build and start the container: +```bash +docker-compose up -d --build +``` + +Note: The `--build` flag ensures the image is rebuilt with the latest code changes. + +2. Access the web interface: +- Main interface: http://localhost:5000 +- Map viewer: http://localhost:5000/map + +3. Stop the container: +```bash +docker-compose down +``` + +### Using Docker Directly + +1. Build the image: +```bash +docker build -t villages-import . +``` + +2. Run the container: +```bash +docker run -d \ + -p 5000:5000 \ + -v "$(pwd)/data:/data" \ + --name villages-import \ + villages-import +``` + +3. View logs: +```bash +docker logs -f villages-import +``` + +4. Stop the container: +```bash +docker stop villages-import +docker rm villages-import +``` + +## Features + +### Main Dashboard (/) +- Run data processing scripts for Lake and Sumter counties +- View real-time script output +- Access to: + - Diff Roads + - Diff Addresses + - Diff Multi-Use Paths + - Download OSM Data + +### Map Viewer (/map) +- Interactive map viewer for GeoJSON files +- Upload and compare OSM, Diff, and County data +- Filter by removed/added features +- Hide highway=service roads +- Drag-and-drop layer reordering +- Click on features to view properties +- Accept/reject diff features + +## Volume Mounts + +The Docker container mounts a single data directory: +- `./data` → `/data` - All data files, organized by date + +Inside `/data`, the structure is: +- `/data/latest/` - Symlink to the most recent data directory +- `/data/YYMMDD/lake/` - Lake County data for that date +- `/data/YYMMDD/sumter/` - Sumter County data for that date + +All changes are persisted on the host in the local `./data` folder. + +## API Endpoints + +- `GET /` - Main dashboard +- `GET /map` - Map viewer +- `POST /api/run-script` - Execute a processing script +- `GET /api/job-status/` - Get script status and logs +- `GET /api/list-files` - List available GeoJSON files +- `GET /data/` - Serve GeoJSON files + +## Troubleshooting + +### Port already in use +If port 5000 is already in use, edit `docker-compose.yml`: +```yaml +ports: + - "8080:5000" # Change 8080 to any available port +``` + +### Permission issues +Ensure the data directory has proper permissions: +```bash +mkdir -p data +chmod -R 755 data +``` + +### View container logs +```bash +docker-compose logs -f +``` + +### Rebuild after code changes +```bash +docker-compose down +docker-compose build --no-cache +docker-compose up -d +``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d8a0057 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3.11-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + libspatialindex-dev \ + libgeos-dev \ + libproj-dev \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy requirements and install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy only necessary Python scripts and web files +COPY *.py ./ +COPY web/ ./web/ + +# Create /data directory (will be mounted as volume) +RUN mkdir -p /data + +# Expose port +EXPOSE 5000 + +# Set environment variables +ENV FLASK_APP=web/server.py +ENV PYTHONUNBUFFERED=1 + +# Run the Flask server +CMD ["python", "web/server.py"] diff --git a/README.md b/README.md index fa230d2..0991956 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,10 @@ See compare-addresses.py for an automated way of running the complete address di - Streets: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/73 - Addresses: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/16 - Highways: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/9 -- Sumter GIS Road Centerlines, Addresses, and Multi Modal Trails is via emailing their GIS team and accessing their Dropbox (https://www.dropbox.com/scl/fo/67nh5y8e42tr2kzdmmcg4/AAsF7Ay0MRUN-e_Ajlh5yWQ?rlkey=h6u606av0d2zkszk9lm3qijlt&e=1&st=7j7i94f8&dl=0) +- Sumter GIS: - Alternately, roads: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/9177e17c72d3433aa79630c7eda84add/about - Addresses: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/c75c5aac13a648968c5596b0665be28b/about + - Email for Multi-Modal Paths. - Marion (TODO) ## Instructions diff --git a/compare-addresses.py b/compare-addresses.py index 7a7fd88..6e9ae63 100644 --- a/compare-addresses.py +++ b/compare-addresses.py @@ -457,28 +457,26 @@ out geom;""" return new_addresses, existing_addresses, removed_addresses - def save_results(self, new_addresses: List[Dict], existing_addresses: List[Dict], + def save_results(self, new_addresses: List[Dict], existing_addresses: List[Dict], removed_addresses: List[Dict], output_dir: str): """Save comparison results to separate GeoJSON files.""" output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) - - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - + # Save new addresses (to add to OSM) if new_addresses: new_gdf = gpd.GeoDataFrame(new_addresses) - new_file = output_dir / f"addresses_to_add_{timestamp}.geojson" + new_file = output_dir / "addresses-to-add.geojson" new_gdf.to_file(new_file, driver='GeoJSON') print(f"Saved {len(new_addresses)} new addresses to {new_file}") - + # Save removed addresses (missing from local data) if removed_addresses: removed_gdf = gpd.GeoDataFrame(removed_addresses) - removed_file = output_dir / f"addresses_potentially_removed_{timestamp}.geojson" + removed_file = output_dir / "addresses-potentially-removed.geojson" removed_gdf.to_file(removed_file, driver='GeoJSON') print(f"Saved {len(removed_addresses)} potentially removed addresses to {removed_file}") - + # Save existing addresses for reference if existing_addresses: # Create simplified format for existing addresses @@ -489,9 +487,9 @@ out geom;""" 'distance_meters': addr['distance_meters'], 'status': 'existing' }) - + existing_gdf = gpd.GeoDataFrame(existing_simple) - existing_file = output_dir / f"addresses_existing_{timestamp}.geojson" + existing_file = output_dir / "addresses-existing.geojson" existing_gdf.to_file(existing_file, driver='GeoJSON') print(f"Saved {len(existing_addresses)} existing addresses to {existing_file}") diff --git a/diff-highways.py b/diff-highways.py index ed75d8a..9886815 100644 --- a/diff-highways.py +++ b/diff-highways.py @@ -594,10 +594,10 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - python compare_roads.py roads1.geojson roads2.geojson - python compare_roads.py roads1.geojson roads2.geojson --tolerance 100 --min-length 200 - python compare_roads.py roads1.geojson roads2.geojson --output differences.geojson - python compare_roads.py roads1.geojson roads2.geojson --jobs 8 --chunk-size 2000 + python diff_highways.py roads1.geojson roads2.geojson + python diff_highways.py roads1.geojson roads2.geojson --tolerance 100 --min-length 200 + python diff_highways.py roads1.geojson roads2.geojson --output differences.geojson + python diff_highways.py roads1.geojson roads2.geojson --jobs 8 --chunk-size 2000 """ ) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..cda1e95 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.8' + +services: + web: + build: + context: . + pull: true + ports: + - "5000:5000" + volumes: + - ./data:/data + environment: + - FLASK_ENV=development + restart: unless-stopped diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1c60d02 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Flask==3.0.0 +numpy<2.0.0 +geopandas>=0.14.0 +pandas>=2.1.0 +shapely>=2.0.0 +pyproj>=3.6.0 +rtree>=1.1.0 diff --git a/web/app.js b/web/app.js index 199571d..d50e6e3 100644 --- a/web/app.js +++ b/web/app.js @@ -11,6 +11,7 @@ let selectedLayer = null; let acceptedFeatures = new Set(); let rejectedFeatures = new Set(); let featurePopup = null; +let layerOrder = ['diff', 'osm', 'county']; // Default layer order (top to bottom) // Initialize map function initMap() { @@ -21,6 +22,16 @@ function initMap() { subdomains: 'abcd', maxZoom: 20 }).addTo(map); + + // Create custom panes for layer ordering + map.createPane('osmPane'); + map.createPane('diffPane'); + map.createPane('countyPane'); + + // Set initial z-indices for panes + map.getPane('osmPane').style.zIndex = 400; + map.getPane('diffPane').style.zIndex = 401; + map.getPane('countyPane').style.zIndex = 402; } // Calculate bounds for all loaded layers @@ -69,7 +80,7 @@ function calculateBounds() { // Style functions function osmStyle(feature) { return { - color: '#8B4513', + color: '#4a4a4a', weight: 3, opacity: 0.7 }; @@ -87,7 +98,7 @@ function diffStyle(feature) { if (rejectedFeatures.has(feature)) { return { - color: '#4a4a4a', + color: '#ff8c00', weight: 3, opacity: 0.8 }; @@ -95,7 +106,7 @@ function diffStyle(feature) { const isRemoved = feature.properties && (feature.properties.removed === true || feature.properties.removed === 'True'); return { - color: isRemoved ? '#dc3545' : '#28a745', + color: isRemoved ? '#ff0000' : '#00ff00', weight: 3, opacity: 0.8 }; @@ -103,12 +114,22 @@ function diffStyle(feature) { function countyStyle(feature) { return { - color: '#800080', + color: '#ff00ff', weight: 3, - opacity: 0.7 + opacity: 0.8 }; } +// Filter function for OSM features +function shouldShowOsmFeature(feature) { + const props = feature.properties || {}; + const isService = props.highway === 'service'; + const hideService = document.getElementById('hideService').checked; + + if (isService && hideService) return false; + return true; +} + // Create layer for OSM data function createOsmLayer() { if (osmLayer) { @@ -118,8 +139,53 @@ function createOsmLayer() { if (!osmData) return; osmLayer = L.geoJSON(osmData, { - style: osmStyle + style: osmStyle, + filter: shouldShowOsmFeature, + pane: 'osmPane', + onEachFeature: function(feature, layer) { + layer.on('click', function(e) { + L.DomEvent.stopPropagation(e); + selectFeature(feature, layer, e, 'osm'); + }); + + layer.on('mouseover', function(e) { + if (selectedLayer !== layer) { + layer.setStyle({ + weight: 5, + opacity: 1 + }); + } + }); + + layer.on('mouseout', function(e) { + if (selectedLayer !== layer) { + layer.setStyle(osmStyle(feature)); + } + }); + } }).addTo(map); + + updateLayerZIndex(); +} + +// Filter function for diff features +function shouldShowFeature(feature) { + const props = feature.properties || {}; + const isRemoved = props.removed === true || props.removed === 'True'; + const isService = props.highway === 'service'; + + const showAdded = document.getElementById('showAdded').checked; + const showRemoved = document.getElementById('showRemoved').checked; + const hideService = document.getElementById('hideService').checked; + + // Check removed/added filter + if (isRemoved && !showRemoved) return false; + if (!isRemoved && !showAdded) return false; + + // Check service filter + if (isService && hideService) return false; + + return true; } // Create layer for diff data with click handlers @@ -132,10 +198,12 @@ function createDiffLayer() { diffLayer = L.geoJSON(diffData, { style: diffStyle, + filter: shouldShowFeature, + pane: 'diffPane', onEachFeature: function(feature, layer) { layer.on('click', function(e) { L.DomEvent.stopPropagation(e); - selectFeature(feature, layer, e); + selectFeature(feature, layer, e, 'diff'); }); layer.on('mouseover', function(e) { @@ -154,6 +222,18 @@ function createDiffLayer() { }); } }).addTo(map); + + updateLayerZIndex(); +} + +// Filter function for county features +function shouldShowCountyFeature(feature) { + const props = feature.properties || {}; + const isService = props.highway === 'service'; + const hideService = document.getElementById('hideService').checked; + + if (isService && hideService) return false; + return true; } // Create layer for county data @@ -165,24 +245,53 @@ function createCountyLayer() { if (!countyData) return; countyLayer = L.geoJSON(countyData, { - style: countyStyle + style: countyStyle, + filter: shouldShowCountyFeature, + pane: 'countyPane', + onEachFeature: function(feature, layer) { + layer.on('click', function(e) { + L.DomEvent.stopPropagation(e); + selectFeature(feature, layer, e, 'county'); + }); + + layer.on('mouseover', function(e) { + if (selectedLayer !== layer) { + layer.setStyle({ + weight: 5, + opacity: 1 + }); + } + }); + + layer.on('mouseout', function(e) { + if (selectedLayer !== layer) { + layer.setStyle(countyStyle(feature)); + } + }); + } }); // County layer is hidden by default if (document.getElementById('countyToggle').checked) { countyLayer.addTo(map); } + + updateLayerZIndex(); } -// Select a feature from diff layer -function selectFeature(feature, layer, e) { +// Select a feature from any layer +function selectFeature(feature, layer, e, layerType = 'diff') { // Deselect previous feature if (selectedLayer) { - selectedLayer.setStyle(diffStyle(selectedLayer.feature)); + // Get the appropriate style function based on previous layer type + const styleFunc = selectedLayer._layerType === 'diff' ? diffStyle : + selectedLayer._layerType === 'osm' ? osmStyle : countyStyle; + selectedLayer.setStyle(styleFunc(selectedLayer.feature)); } selectedFeature = feature; selectedLayer = layer; + selectedLayer._layerType = layerType; // Store layer type for later layer.setStyle({ weight: 6, opacity: 1, @@ -196,12 +305,29 @@ function selectFeature(feature, layer, e) { const isRejected = rejectedFeatures.has(feature); let html = '
'; - html += `
Status: ${isRemoved ? 'Removed' : 'Added/Modified'}
`; - // Display all non-null properties + // Show layer type + html += `
Layer: ${layerType.toUpperCase()}
`; + + // Only show status for diff layer + if (layerType === 'diff') { + html += `
Status: ${isRemoved ? 'Removed' : 'Added/Modified'}
`; + } + + // Display all non-null properties with custom ordering const displayProps = Object.entries(props) .filter(([key, value]) => value !== null && value !== undefined && key !== 'removed') - .sort(([a], [b]) => a.localeCompare(b)); + .sort(([a], [b]) => { + // Priority order: name, highway, then alphabetical + const priorityOrder = { 'name': 0, 'highway': 1 }; + const aPriority = priorityOrder[a] ?? 999; + const bPriority = priorityOrder[b] ?? 999; + + if (aPriority !== bPriority) { + return aPriority - bPriority; + } + return a.localeCompare(b); + }); if (displayProps.length > 0) { html += '
'; @@ -211,16 +337,20 @@ function selectFeature(feature, layer, e) { html += '
'; } - if (isAccepted) { - html += '
✓ Accepted
'; - } else if (isRejected) { - html += '
✗ Rejected
'; + // Only show accept/reject for diff layer + if (layerType === 'diff') { + if (isAccepted) { + html += '
✓ Accepted
'; + } else if (isRejected) { + html += '
✗ Rejected
'; + } + + html += '
'; + html += ''; + html += ''; + html += '
'; } - html += '
'; - html += ''; - html += ''; - html += '
'; html += '
'; // Remove old popup if exists @@ -301,6 +431,10 @@ function rejectFeature() { showStatus(`${acceptedFeatures.size} accepted, ${rejectedFeatures.size} rejected`, 'success'); } +// Expose functions globally for onclick handlers +window.acceptFeature = acceptFeature; +window.rejectFeature = rejectFeature; + // Update save button state function updateSaveButton() { document.getElementById('saveButton').disabled = @@ -417,6 +551,24 @@ function showStatus(message, type) { }, 3000); } +// Update pane z-index based on order +function updateLayerZIndex() { + const panes = { + 'osm': 'osmPane', + 'diff': 'diffPane', + 'county': 'countyPane' + }; + + // Reverse index so first item in list is on top + layerOrder.forEach((layerName, index) => { + const paneName = panes[layerName]; + const pane = map.getPane(paneName); + if (pane) { + pane.style.zIndex = 400 + (layerOrder.length - 1 - index); + } + }); +} + // Toggle layer visibility function toggleLayer(layerId, layer) { const checkbox = document.getElementById(layerId); @@ -424,6 +576,7 @@ function toggleLayer(layerId, layer) { if (checkbox.checked && layer) { if (!map.hasLayer(layer)) { map.addLayer(layer); + updateLayerZIndex(); } } else if (layer) { if (map.hasLayer(layer)) { @@ -449,9 +602,82 @@ document.addEventListener('DOMContentLoaded', function() { toggleLayer('countyToggle', countyLayer); }); + // Diff filter toggles + document.getElementById('showAdded').addEventListener('change', function() { + createDiffLayer(); + }); + + document.getElementById('showRemoved').addEventListener('change', function() { + createDiffLayer(); + }); + + document.getElementById('hideService').addEventListener('change', function() { + createDiffLayer(); + createOsmLayer(); + createCountyLayer(); + }); + // Load button document.getElementById('loadButton').addEventListener('click', loadFiles); // Save button document.getElementById('saveButton').addEventListener('click', saveAcceptedItems); + + // Drag and drop for layer reordering + const layerList = document.getElementById('layerList'); + const layerItems = layerList.querySelectorAll('.layer-item'); + + let draggedElement = null; + + layerItems.forEach(item => { + item.addEventListener('dragstart', function(e) { + draggedElement = this; + this.classList.add('dragging'); + e.dataTransfer.effectAllowed = 'move'; + }); + + item.addEventListener('dragend', function(e) { + this.classList.remove('dragging'); + draggedElement = null; + }); + + item.addEventListener('dragover', function(e) { + e.preventDefault(); + e.dataTransfer.dropEffect = 'move'; + + if (this === draggedElement) return; + + const afterElement = getDragAfterElement(layerList, e.clientY); + if (afterElement == null) { + layerList.appendChild(draggedElement); + } else { + layerList.insertBefore(draggedElement, afterElement); + } + }); + + item.addEventListener('drop', function(e) { + e.preventDefault(); + + // Update layer order based on new DOM order + layerOrder = Array.from(layerList.querySelectorAll('.layer-item')) + .map(item => item.dataset.layer); + + updateLayerZIndex(); + }); + }); + + function getDragAfterElement(container, y) { + const draggableElements = [...container.querySelectorAll('.layer-item:not(.dragging)')]; + + return draggableElements.reduce((closest, child) => { + const box = child.getBoundingClientRect(); + const offset = y - box.top - box.height / 2; + + if (offset < 0 && offset > closest.offset) { + return { offset: offset, element: child }; + } else { + return closest; + } + }, { offset: Number.NEGATIVE_INFINITY }).element; + } }); diff --git a/web/index.html b/web/index.html index b0f2d88..766e13e 100644 --- a/web/index.html +++ b/web/index.html @@ -50,6 +50,25 @@ font-size: 13px; } + .layer-item { + display: flex; + align-items: center; + margin: 8px 0; + padding: 5px; + background: #f8f9fa; + border-radius: 4px; + cursor: move; + font-size: 13px; + } + + .layer-item.dragging { + opacity: 0.5; + } + + .layer-item input[type="checkbox"] { + margin-right: 8px; + } + .controls input[type="checkbox"] { margin-right: 8px; } @@ -132,29 +151,45 @@
-

Layer Controls

+

Layer Controls (top to bottom)

+
+
+ + Diff Layer +
+
+ + OSM Roads (Gray) +
+
+ + County Layer (Purple) +
+
+ +

Diff Filters

Load Files

-
- - -
+
+ + +
diff --git a/web/server.py b/web/server.py new file mode 100644 index 0000000..e7eeda5 --- /dev/null +++ b/web/server.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +Flask web server for The Villages Import Tools +""" +from flask import Flask, render_template, jsonify, request, send_from_directory +import subprocess +import os +import threading +import json +from datetime import datetime + +app = Flask(__name__, static_folder='static', template_folder='templates') + +# Store running processes +running_processes = {} +process_logs = {} + +@app.route('/') +def index(): + """Main index page with script execution buttons""" + # Get available scripts and organize them + script_map = get_script_map() + + # Organize scripts by category + scripts_by_category = { + 'Download County Data': ['download-county-addresses', 'download-county-roads', 'download-county-paths'], + 'Download OSM Data': ['download-osm-roads', 'download-osm-paths'], + 'Convert Data': ['convert-roads', 'convert-paths'], + 'Diff Data': ['diff-roads', 'diff-paths', 'diff-addresses'], + 'Utilities': ['ls', 'make-new-latest'] + } + + return render_template('index.html', + script_map=script_map, + scripts_by_category=scripts_by_category) + +@app.route('/map') +def map_viewer(): + """Map viewer page""" + return render_template('map.html') + +def get_script_map(): + """Get the map of available scripts and their commands""" + return { + 'ls': 'ls -alr /data', + 'make-new-latest': 'cd /data && NEWDIR=$(date +%y%m%d) && mkdir -p $NEWDIR/lake $NEWDIR/sumter && ln -sfn $NEWDIR latest', + # todo: make a clean-old-data script + 'download-county-addresses': { + # deliver files with standardized names + 'sumter': 'mkdir -p /data/latest/sumter && wget https://www.arcgis.com/sharing/rest/content/items/c75c5aac13a648968c5596b0665be28b/data -O /data/latest/sumter/addresses.shp.zip', + 'lake': 'mkdir -p /data/latest/lake && wget [LAKE_URL_HERE] -O /data/latest/lake/addresses.shp.zip' + }, + 'download-county-roads': { + # deliver files with standardized names + 'sumter': 'mkdir -p /data/latest/sumter && wget https://www.arcgis.com/sharing/rest/content/items/9177e17c72d3433aa79630c7eda84add/data -O /data/latest/sumter/roads.shp.zip', + 'lake': 'mkdir -p /data/latest/lake && wget [LAKE_URL_HERE] -O /data/latest/lake/roads.shp.zip' + }, + 'download-county-paths': { + # deliver files with standardized names + #'sumter': ['/data/latest/sumter/paths.shp.zip'], + #'lake': ['/data/latest/lake/paths.shp.zip'] + }, + # todo: integrate osm downloading and shapefile converting into diff-roads like addresses + 'download-osm-roads': { + 'lake': ['python', 'download-overpass.py', '--type', 'highways', 'Lake County', 'Florida', '/data/latest/lake/osm-roads.geojson'], + 'sumter': ['python', 'download-overpass.py', '--type', 'highways', 'Sumter County', 'Florida', '/data/latest/sumter/osm-roads.geojson'] + }, + 'download-osm-paths': { + # todo: no lake county paths + #'lake': ['python', 'download-overpass.py', '--type', 'highways', 'Lake County', 'Florida', '/data/latest/lake/osm-roads.geojson'], + 'sumter': ['python', 'download-overpass.py', '--type', 'paths', 'Sumter County', 'Florida', '/data/latest/sumter/osm-paths.geojson'] + }, + # todo + 'convert-roads': { + 'sumter': ['python', 'shp-to-geojson.py', '/data/latest/sumter/roads.shp.zip', '/data/latest/sumter/county-roads.geojson'], + 'lake': ['python', 'shp-to-geojson.py', '/data/latest/lake/roads.shp.zip', '/data/latest/lake/county-roads.geojson'] + }, + 'convert-paths': { + #todo: delete sumter-multi-modal-convert.py ? + 'sumter': ['python', 'shp-to-geojson.py', '/data/latest/sumter/paths.shp.zip', '/data/latest/sumter/county-paths.geojson'], + }, + 'diff-roads': { + 'lake': ['python', 'diff-highways.py', '/data/latest/lake/osm-roads.geojson', '/data/latest/lake/county-roads.geojson', '--output', '/data/latest/lake/diff-roads.geojson'], + 'sumter': ['python', 'diff-highways.py', '/data/latest/sumter/osm-roads.geojson', '/data/latest/sumter/county-roads.geojson', '--output', '/data/latest/sumter/diff-roads.geojson'] + }, + 'diff-paths': { + #todo: no lake county data for paths + #'lake': ['python', 'diff-highways.py', '/data/latest/lake/osm-paths.geojson', '/data/latest/lake/county-paths.geojson', '--output', '/data/latest/lake/diff-paths.geojson'], + 'sumter': ['python', 'diff-highways.py', '/data/latest/sumter/osm-paths.geojson', '/data/latest/sumter/county-paths.geojson', '--output', '/data/latest/sumter/diff-paths.geojson'], + }, + # addresses need no osm download or shapefile convert, just county download + 'diff-addresses': { + #todo: delete sumter-address-convert.py ? + 'lake': ['python', 'compare-addresses.py', 'Lake', 'Florida', '--local-zip', '/data/latest/lake/addresses.shp.zip', '--output-dir', '/data/latest/lake', '--cache-dir', '/data/osm_cache'], + 'sumter': ['python', 'compare-addresses.py', 'Sumter', 'Florida', '--local-zip', '/data/latest/sumter/addresses.shp.zip', '--output-dir', '/data/latest/sumter', '--cache-dir', '/data/osm_cache'] + }, + } + +@app.route('/api/run-script', methods=['POST']) +def run_script(): + """Execute a script in the background""" + data = request.json + script_name = data.get('script') + county = data.get('county', '') + + if not script_name: + return jsonify({'error': 'No script specified'}), 400 + + script_map = get_script_map() + + if script_name not in script_map: + return jsonify({'error': 'Unknown script'}), 400 + + script_config = script_map[script_name] + + # Handle both string commands and dict of county-specific commands + if isinstance(script_config, str): + # Simple string command (like 'ls') + cmd = ['bash', '-c', script_config] + elif isinstance(script_config, dict): + # County-specific commands + if not county: + return jsonify({'error': 'County required for this script'}), 400 + if county not in script_config: + return jsonify({'error': f'County {county} not supported for {script_name}'}), 400 + + cmd_config = script_config[county] + if isinstance(cmd_config, str): + cmd = ['bash', '-c', cmd_config] + else: + cmd = cmd_config + else: + return jsonify({'error': 'Invalid script configuration'}), 400 + + # Generate a unique job ID + job_id = f"{script_name}_{county}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + # Start process in background + def run_command(): + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + cwd=os.path.dirname(os.path.dirname(__file__)) + ) + running_processes[job_id] = process + process_logs[job_id] = [] + + # Stream output + for line in process.stdout: + process_logs[job_id].append(line) + + process.wait() + process_logs[job_id].append(f"\n[Process completed with exit code {process.returncode}]") + + except Exception as e: + process_logs[job_id].append(f"\n[ERROR: {str(e)}]") + finally: + if job_id in running_processes: + del running_processes[job_id] + + thread = threading.Thread(target=run_command) + thread.daemon = True + thread.start() + + return jsonify({'job_id': job_id, 'status': 'started'}) + +@app.route('/api/job-status/') +def job_status(job_id): + """Get status and logs for a job""" + is_running = job_id in running_processes + logs = process_logs.get(job_id, []) + + return jsonify({ + 'job_id': job_id, + 'running': is_running, + 'logs': logs + }) + +@app.route('/api/list-files') +def list_files(): + """List available GeoJSON files""" + data_dir = '/data' + + files = { + 'diff': [], + 'osm': [], + 'county': [] + } + + # Scan directories for geojson files + if os.path.exists(data_dir): + for root, dirs, filenames in os.walk(data_dir): + for filename in filenames: + if filename.endswith('.geojson'): + rel_path = os.path.relpath(os.path.join(root, filename), data_dir) + + if 'diff' in filename.lower(): + files['diff'].append(rel_path) + elif 'osm' in filename.lower(): + files['osm'].append(rel_path) + elif any(county in filename.lower() for county in ['lake', 'sumter']): + files['county'].append(rel_path) + + return jsonify(files) + +@app.route('/data/') +def serve_data(filename): + """Serve GeoJSON files""" + return send_from_directory('/data', filename) + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000, debug=True) diff --git a/web/static/map.js b/web/static/map.js new file mode 100644 index 0000000..d50e6e3 --- /dev/null +++ b/web/static/map.js @@ -0,0 +1,683 @@ +// Global state +let map; +let osmLayer; +let diffLayer; +let countyLayer; +let osmData = null; +let diffData = null; +let countyData = null; +let selectedFeature = null; +let selectedLayer = null; +let acceptedFeatures = new Set(); +let rejectedFeatures = new Set(); +let featurePopup = null; +let layerOrder = ['diff', 'osm', 'county']; // Default layer order (top to bottom) + +// Initialize map +function initMap() { + map = L.map('map').setView([28.7, -81.7], 12); + + L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { + attribution: '© OpenStreetMap contributors © CARTO', + subdomains: 'abcd', + maxZoom: 20 + }).addTo(map); + + // Create custom panes for layer ordering + map.createPane('osmPane'); + map.createPane('diffPane'); + map.createPane('countyPane'); + + // Set initial z-indices for panes + map.getPane('osmPane').style.zIndex = 400; + map.getPane('diffPane').style.zIndex = 401; + map.getPane('countyPane').style.zIndex = 402; +} + +// Calculate bounds for all loaded layers +function calculateBounds() { + const bounds = L.latLngBounds([]); + let hasData = false; + + if (osmData && osmData.features.length > 0) { + L.geoJSON(osmData).eachLayer(layer => { + if (layer.getBounds) { + bounds.extend(layer.getBounds()); + } else if (layer.getLatLng) { + bounds.extend(layer.getLatLng()); + } + }); + hasData = true; + } + + if (diffData && diffData.features.length > 0) { + L.geoJSON(diffData).eachLayer(layer => { + if (layer.getBounds) { + bounds.extend(layer.getBounds()); + } else if (layer.getLatLng) { + bounds.extend(layer.getLatLng()); + } + }); + hasData = true; + } + + if (countyData && countyData.features.length > 0) { + L.geoJSON(countyData).eachLayer(layer => { + if (layer.getBounds) { + bounds.extend(layer.getBounds()); + } else if (layer.getLatLng) { + bounds.extend(layer.getLatLng()); + } + }); + hasData = true; + } + + if (hasData && bounds.isValid()) { + map.fitBounds(bounds, { padding: [50, 50] }); + } +} + +// Style functions +function osmStyle(feature) { + return { + color: '#4a4a4a', + weight: 3, + opacity: 0.7 + }; +} + +function diffStyle(feature) { + // Check if feature is accepted or rejected + if (acceptedFeatures.has(feature)) { + return { + color: '#007bff', + weight: 3, + opacity: 0.8 + }; + } + + if (rejectedFeatures.has(feature)) { + return { + color: '#ff8c00', + weight: 3, + opacity: 0.8 + }; + } + + const isRemoved = feature.properties && (feature.properties.removed === true || feature.properties.removed === 'True'); + return { + color: isRemoved ? '#ff0000' : '#00ff00', + weight: 3, + opacity: 0.8 + }; +} + +function countyStyle(feature) { + return { + color: '#ff00ff', + weight: 3, + opacity: 0.8 + }; +} + +// Filter function for OSM features +function shouldShowOsmFeature(feature) { + const props = feature.properties || {}; + const isService = props.highway === 'service'; + const hideService = document.getElementById('hideService').checked; + + if (isService && hideService) return false; + return true; +} + +// Create layer for OSM data +function createOsmLayer() { + if (osmLayer) { + map.removeLayer(osmLayer); + } + + if (!osmData) return; + + osmLayer = L.geoJSON(osmData, { + style: osmStyle, + filter: shouldShowOsmFeature, + pane: 'osmPane', + onEachFeature: function(feature, layer) { + layer.on('click', function(e) { + L.DomEvent.stopPropagation(e); + selectFeature(feature, layer, e, 'osm'); + }); + + layer.on('mouseover', function(e) { + if (selectedLayer !== layer) { + layer.setStyle({ + weight: 5, + opacity: 1 + }); + } + }); + + layer.on('mouseout', function(e) { + if (selectedLayer !== layer) { + layer.setStyle(osmStyle(feature)); + } + }); + } + }).addTo(map); + + updateLayerZIndex(); +} + +// Filter function for diff features +function shouldShowFeature(feature) { + const props = feature.properties || {}; + const isRemoved = props.removed === true || props.removed === 'True'; + const isService = props.highway === 'service'; + + const showAdded = document.getElementById('showAdded').checked; + const showRemoved = document.getElementById('showRemoved').checked; + const hideService = document.getElementById('hideService').checked; + + // Check removed/added filter + if (isRemoved && !showRemoved) return false; + if (!isRemoved && !showAdded) return false; + + // Check service filter + if (isService && hideService) return false; + + return true; +} + +// Create layer for diff data with click handlers +function createDiffLayer() { + if (diffLayer) { + map.removeLayer(diffLayer); + } + + if (!diffData) return; + + diffLayer = L.geoJSON(diffData, { + style: diffStyle, + filter: shouldShowFeature, + pane: 'diffPane', + onEachFeature: function(feature, layer) { + layer.on('click', function(e) { + L.DomEvent.stopPropagation(e); + selectFeature(feature, layer, e, 'diff'); + }); + + layer.on('mouseover', function(e) { + if (selectedLayer !== layer) { + layer.setStyle({ + weight: 5, + opacity: 1 + }); + } + }); + + layer.on('mouseout', function(e) { + if (selectedLayer !== layer) { + layer.setStyle(diffStyle(feature)); + } + }); + } + }).addTo(map); + + updateLayerZIndex(); +} + +// Filter function for county features +function shouldShowCountyFeature(feature) { + const props = feature.properties || {}; + const isService = props.highway === 'service'; + const hideService = document.getElementById('hideService').checked; + + if (isService && hideService) return false; + return true; +} + +// Create layer for county data +function createCountyLayer() { + if (countyLayer) { + map.removeLayer(countyLayer); + } + + if (!countyData) return; + + countyLayer = L.geoJSON(countyData, { + style: countyStyle, + filter: shouldShowCountyFeature, + pane: 'countyPane', + onEachFeature: function(feature, layer) { + layer.on('click', function(e) { + L.DomEvent.stopPropagation(e); + selectFeature(feature, layer, e, 'county'); + }); + + layer.on('mouseover', function(e) { + if (selectedLayer !== layer) { + layer.setStyle({ + weight: 5, + opacity: 1 + }); + } + }); + + layer.on('mouseout', function(e) { + if (selectedLayer !== layer) { + layer.setStyle(countyStyle(feature)); + } + }); + } + }); + + // County layer is hidden by default + if (document.getElementById('countyToggle').checked) { + countyLayer.addTo(map); + } + + updateLayerZIndex(); +} + +// Select a feature from any layer +function selectFeature(feature, layer, e, layerType = 'diff') { + // Deselect previous feature + if (selectedLayer) { + // Get the appropriate style function based on previous layer type + const styleFunc = selectedLayer._layerType === 'diff' ? diffStyle : + selectedLayer._layerType === 'osm' ? osmStyle : countyStyle; + selectedLayer.setStyle(styleFunc(selectedLayer.feature)); + } + + selectedFeature = feature; + selectedLayer = layer; + selectedLayer._layerType = layerType; // Store layer type for later + layer.setStyle({ + weight: 6, + opacity: 1, + color: '#ffc107' + }); + + // Create popup near the clicked location + const props = feature.properties || {}; + const isRemoved = props.removed === true || props.removed === 'True'; + const isAccepted = acceptedFeatures.has(feature); + const isRejected = rejectedFeatures.has(feature); + + let html = '
'; + + // Show layer type + html += `
Layer: ${layerType.toUpperCase()}
`; + + // Only show status for diff layer + if (layerType === 'diff') { + html += `
Status: ${isRemoved ? 'Removed' : 'Added/Modified'}
`; + } + + // Display all non-null properties with custom ordering + const displayProps = Object.entries(props) + .filter(([key, value]) => value !== null && value !== undefined && key !== 'removed') + .sort(([a], [b]) => { + // Priority order: name, highway, then alphabetical + const priorityOrder = { 'name': 0, 'highway': 1 }; + const aPriority = priorityOrder[a] ?? 999; + const bPriority = priorityOrder[b] ?? 999; + + if (aPriority !== bPriority) { + return aPriority - bPriority; + } + return a.localeCompare(b); + }); + + if (displayProps.length > 0) { + html += '
'; + for (const [key, value] of displayProps) { + html += `
${key}: ${value}
`; + } + html += '
'; + } + + // Only show accept/reject for diff layer + if (layerType === 'diff') { + if (isAccepted) { + html += '
✓ Accepted
'; + } else if (isRejected) { + html += '
✗ Rejected
'; + } + + html += '
'; + html += ''; + html += ''; + html += '
'; + } + + html += '
'; + + // Remove old popup if exists + if (featurePopup) { + map.closePopup(featurePopup); + } + + // Create popup at click location + featurePopup = L.popup({ + maxWidth: 300, + closeButton: true, + autoClose: false, + closeOnClick: false + }) + .setLatLng(e.latlng) + .setContent(html) + .openOn(map); + + // Handle popup close + featurePopup.on('remove', function() { + if (selectedLayer) { + selectedLayer.setStyle(diffStyle(selectedLayer.feature)); + selectedLayer = null; + selectedFeature = null; + } + }); +} + +// Accept a feature +function acceptFeature() { + if (!selectedFeature) return; + + // Remove from rejected if present + rejectedFeatures.delete(selectedFeature); + + // Add to accepted + acceptedFeatures.add(selectedFeature); + + // Update layer style + if (selectedLayer) { + selectedLayer.setStyle(diffStyle(selectedFeature)); + } + + // Close popup + if (featurePopup) { + map.closePopup(featurePopup); + } + + // Enable save button + updateSaveButton(); + + showStatus(`${acceptedFeatures.size} accepted, ${rejectedFeatures.size} rejected`, 'success'); +} + +// Reject a feature +function rejectFeature() { + if (!selectedFeature) return; + + // Remove from accepted if present + acceptedFeatures.delete(selectedFeature); + + // Add to rejected + rejectedFeatures.add(selectedFeature); + + // Update layer style + if (selectedLayer) { + selectedLayer.setStyle(diffStyle(selectedFeature)); + } + + // Close popup + if (featurePopup) { + map.closePopup(featurePopup); + } + + // Enable save button + updateSaveButton(); + + showStatus(`${acceptedFeatures.size} accepted, ${rejectedFeatures.size} rejected`, 'success'); +} + +// Expose functions globally for onclick handlers +window.acceptFeature = acceptFeature; +window.rejectFeature = rejectFeature; + +// Update save button state +function updateSaveButton() { + document.getElementById('saveButton').disabled = + acceptedFeatures.size === 0 && rejectedFeatures.size === 0; +} + +// Load file from input +function loadFile(input) { + return new Promise((resolve, reject) => { + const file = input.files[0]; + if (!file) { + resolve(null); + return; + } + + const reader = new FileReader(); + reader.onload = (e) => { + try { + const data = JSON.parse(e.target.result); + resolve(data); + } catch (error) { + reject(new Error(`Failed to parse ${file.name}: ${error.message}`)); + } + }; + reader.onerror = () => reject(new Error(`Failed to read ${file.name}`)); + reader.readAsText(file); + }); +} + +// Load all files +async function loadFiles() { + try { + showStatus('Loading files...', 'success'); + + const osmInput = document.getElementById('osmFile'); + const diffInput = document.getElementById('diffFile'); + const countyInput = document.getElementById('countyFile'); + + // Load files + osmData = await loadFile(osmInput); + diffData = await loadFile(diffInput); + countyData = await loadFile(countyInput); + + if (!osmData && !diffData && !countyData) { + showStatus('Please select at least one file', 'error'); + return; + } + + // Create layers + createOsmLayer(); + createDiffLayer(); + createCountyLayer(); + + // Fit bounds to smallest layer + calculateBounds(); + + showStatus('Files loaded successfully!', 'success'); + + // Enable save button if we have diff data + document.getElementById('saveButton').disabled = !diffData; + + } catch (error) { + showStatus(error.message, 'error'); + console.error(error); + } +} + +// Save accepted and rejected items to original diff file +async function saveAcceptedItems() { + if (!diffData || (acceptedFeatures.size === 0 && rejectedFeatures.size === 0)) { + showStatus('No features to save', 'error'); + return; + } + + try { + // Add accepted=true or accepted=false property to features + diffData.features.forEach(feature => { + if (acceptedFeatures.has(feature)) { + feature.properties.accepted = true; + } else if (rejectedFeatures.has(feature)) { + feature.properties.accepted = false; + } + }); + + // Create download + const dataStr = JSON.stringify(diffData, null, 2); + const dataBlob = new Blob([dataStr], { type: 'application/json' }); + const url = URL.createObjectURL(dataBlob); + + const link = document.createElement('a'); + link.href = url; + link.download = 'diff-updated.geojson'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); + + showStatus(`Saved ${acceptedFeatures.size} accepted, ${rejectedFeatures.size} rejected`, 'success'); + + } catch (error) { + showStatus(`Save failed: ${error.message}`, 'error'); + console.error(error); + } +} + +// Show status message +function showStatus(message, type) { + const status = document.getElementById('status'); + status.textContent = message; + status.className = `status ${type}`; + + setTimeout(() => { + status.classList.add('hidden'); + }, 3000); +} + +// Update pane z-index based on order +function updateLayerZIndex() { + const panes = { + 'osm': 'osmPane', + 'diff': 'diffPane', + 'county': 'countyPane' + }; + + // Reverse index so first item in list is on top + layerOrder.forEach((layerName, index) => { + const paneName = panes[layerName]; + const pane = map.getPane(paneName); + if (pane) { + pane.style.zIndex = 400 + (layerOrder.length - 1 - index); + } + }); +} + +// Toggle layer visibility +function toggleLayer(layerId, layer) { + const checkbox = document.getElementById(layerId); + + if (checkbox.checked && layer) { + if (!map.hasLayer(layer)) { + map.addLayer(layer); + updateLayerZIndex(); + } + } else if (layer) { + if (map.hasLayer(layer)) { + map.removeLayer(layer); + } + } +} + +// Event listeners +document.addEventListener('DOMContentLoaded', function() { + initMap(); + + // Layer toggles + document.getElementById('osmToggle').addEventListener('change', function() { + toggleLayer('osmToggle', osmLayer); + }); + + document.getElementById('diffToggle').addEventListener('change', function() { + toggleLayer('diffToggle', diffLayer); + }); + + document.getElementById('countyToggle').addEventListener('change', function() { + toggleLayer('countyToggle', countyLayer); + }); + + // Diff filter toggles + document.getElementById('showAdded').addEventListener('change', function() { + createDiffLayer(); + }); + + document.getElementById('showRemoved').addEventListener('change', function() { + createDiffLayer(); + }); + + document.getElementById('hideService').addEventListener('change', function() { + createDiffLayer(); + createOsmLayer(); + createCountyLayer(); + }); + + // Load button + document.getElementById('loadButton').addEventListener('click', loadFiles); + + // Save button + document.getElementById('saveButton').addEventListener('click', saveAcceptedItems); + + // Drag and drop for layer reordering + const layerList = document.getElementById('layerList'); + const layerItems = layerList.querySelectorAll('.layer-item'); + + let draggedElement = null; + + layerItems.forEach(item => { + item.addEventListener('dragstart', function(e) { + draggedElement = this; + this.classList.add('dragging'); + e.dataTransfer.effectAllowed = 'move'; + }); + + item.addEventListener('dragend', function(e) { + this.classList.remove('dragging'); + draggedElement = null; + }); + + item.addEventListener('dragover', function(e) { + e.preventDefault(); + e.dataTransfer.dropEffect = 'move'; + + if (this === draggedElement) return; + + const afterElement = getDragAfterElement(layerList, e.clientY); + if (afterElement == null) { + layerList.appendChild(draggedElement); + } else { + layerList.insertBefore(draggedElement, afterElement); + } + }); + + item.addEventListener('drop', function(e) { + e.preventDefault(); + + // Update layer order based on new DOM order + layerOrder = Array.from(layerList.querySelectorAll('.layer-item')) + .map(item => item.dataset.layer); + + updateLayerZIndex(); + }); + }); + + function getDragAfterElement(container, y) { + const draggableElements = [...container.querySelectorAll('.layer-item:not(.dragging)')]; + + return draggableElements.reduce((closest, child) => { + const box = child.getBoundingClientRect(); + const offset = y - box.top - box.height / 2; + + if (offset < 0 && offset > closest.offset) { + return { offset: offset, element: child }; + } else { + return closest; + } + }, { offset: Number.NEGATIVE_INFINITY }).element; + } +}); diff --git a/web/templates/index.html b/web/templates/index.html new file mode 100644 index 0000000..31fb0a8 --- /dev/null +++ b/web/templates/index.html @@ -0,0 +1,311 @@ + + + + + + The Villages Import Tools + + + +
+

The Villages Import Tools

+

Run data processing scripts and view results

+ +
+

Map Viewer

+ Open GeoJSON Map Viewer +
+ +
+

Data Processing Scripts

+ + {% for category, script_names in scripts_by_category.items() %} +
+

{{ category }}

+ + {% for script_name in script_names %} + {% if script_name in script_map %} + {% set script_config = script_map[script_name] %} + + {% if script_config is string %} + {# Simple command with no county selection #} +
+ +
+ {% elif script_config is mapping %} + {# County-specific commands #} +
+ {% if 'lake' in script_config %} + + {% endif %} + {% if 'sumter' in script_config %} + + {% endif %} +
+ {% endif %} + {% endif %} + {% endfor %} +
+ {% endfor %} +
+ +
+

Script Output

+
+
+
+
+ + + + diff --git a/web/templates/map.html b/web/templates/map.html new file mode 100644 index 0000000..bd606df --- /dev/null +++ b/web/templates/map.html @@ -0,0 +1,207 @@ + + + + + + GeoJSON Map Viewer + + + + +
+ +
+

Layer Controls (top to bottom)

+
+
+ + Diff Layer +
+
+ + OSM Roads (Gray) +
+
+ + County Layer (Purple) +
+
+ +

Diff Filters

+ + + + +

Load Files

+
+ + +
+
+ + +
+
+ + +
+ + + + + +
+ + + + +