From 5bec6deb192fbbcb8e81865c5713073022cca067 Mon Sep 17 00:00:00 2001 From: zyphlar Date: Thu, 22 Oct 2026 11:31:34 -0700 Subject: [PATCH] Fold GeoJSON conversion into the county download steps and remove the unused Force redownload checkbox --- README.md | 7 ++-- web/server.py | 24 ++++++------- web/templates/index.html | 76 +++++++++++----------------------------- 3 files changed, 36 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 16b4dcc..a11d674 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,11 @@ ## Using -For each county, run the four steps in order using the dashboard. Each step must complete before the next: +For each county, run the three steps in order using the dashboard. Each step must complete before the next: -1. **Download county data** — fetches the county dataset (or upload a shapefile ZIP for paths) +1. **Download & convert county data** — fetches the county dataset (or upload a shapefile ZIP for paths) and converts it to OSM-tagged GeoJSON 2. **Download OSM data** — queries Overpass for the relevant features in the county -3. **Convert to GeoJSON** — converts the county data to OSM-tagged GeoJSON -4. **Diff vs OSM** — compares county data against OSM; produces added (green) and removed (red) GeoJSON +3. **Diff vs OSM** — compares county data against OSM; produces added (green) and removed (red) GeoJSON Open the result in the Map Viewer. Green = features in county data missing from OSM (candidates to add). Red = features in OSM not in county data. Hide the OSM (gray) and county (purple) layers to focus on the diff. Click a feature to inspect its properties and accept or exclude it. Open accepted features in JOSM for final review before uploading. diff --git a/web/server.py b/web/server.py index 8466f36..5490a80 100644 --- a/web/server.py +++ b/web/server.py @@ -481,7 +481,7 @@ def get_script_map(): dl_roads = {}; dl_addrs = {}; dl_paths = {} osm_roads = {}; osm_addrs = {}; osm_paths = {} - conv_roads = {}; conv_addrs = {}; conv_paths = {} + conv_paths = {} diff_roads = {}; diff_addrs = {}; diff_paths = {} city_report = {} @@ -491,26 +491,29 @@ def get_script_map(): state = c['state'] base = f'/data/{cid}' + # Each county download also converts the shapefile to GeoJSON, so the + # diff can run straight after it. Uploaded paths are converted by a + # separate convert-paths job that the dashboard starts after the upload. + conv_roads = f'python shp-to-geojson.py {base}/roads.shp.zip {base}/county-roads.geojson' + conv_addrs = f'python convert-addresses.py {base}/addresses.shp.zip {base}/county-addresses.geojson' + conv_paths_cmd = f'python shp-to-geojson.py {base}/paths.shp.zip {base}/county-paths.geojson' + if c.get('roads_url'): - dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip' + dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip && {conv_roads}' if c.get('addresses_url'): - dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip' + dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip && {conv_addrs}' pu = c.get('paths_url', '') if pu == 'upload': dl_paths[cid] = {'type': 'upload', 'dest': f'{base}/paths.shp.zip'} + conv_paths[cid] = conv_paths_cmd elif pu: - dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip' + dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip && {conv_paths_cmd}' osm_roads[cid] = ['python', 'download-overpass.py', '--type', 'highways', name, state, f'{base}/osm-roads.geojson'] osm_addrs[cid] = ['python', 'download-overpass.py', '--type', 'addresses', name, state, f'{base}/osm-addresses.geojson'] if pu: osm_paths[cid] = ['python', 'download-overpass.py', '--type', 'paths', name, state, f'{base}/osm-paths.geojson'] - conv_roads[cid] = ['python', 'shp-to-geojson.py', f'{base}/roads.shp.zip', f'{base}/county-roads.geojson'] - conv_addrs[cid] = ['python', 'convert-addresses.py', f'{base}/addresses.shp.zip', f'{base}/county-addresses.geojson'] - if pu: - conv_paths[cid] = ['python', 'shp-to-geojson.py', f'{base}/paths.shp.zip', f'{base}/county-paths.geojson'] - diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson'] diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base] city_report[cid] = ['python', 'generate-city-report.py', '--existing-file', f'{base}/addresses-existing.geojson', '--output-dir', base] @@ -523,8 +526,6 @@ def get_script_map(): sm['download-osm-roads'] = osm_roads sm['download-osm-addresses'] = osm_addrs if osm_paths: sm['download-osm-paths'] = osm_paths - sm['convert-roads'] = conv_roads - sm['convert-addresses'] = conv_addrs if conv_paths: sm['convert-paths'] = conv_paths sm['diff-roads'] = diff_roads sm['diff-addresses'] = diff_addrs @@ -552,7 +553,6 @@ def run_script(): data = request.json script_name = data.get('script') county = data.get('county', '') - force_download = data.get('forceDownload', False) if not script_name: return jsonify({'error': 'No script specified'}), 400 diff --git a/web/templates/index.html b/web/templates/index.html index 7e00f25..aadb7c2 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -215,17 +215,6 @@ .left .btn-run { background: var(--active-color, #28a745); } - /* Force redownload */ - .force-label { - font-size: 10px; - color: #aaa; - display: flex; - align-items: center; - gap: 3px; - cursor: pointer; - white-space: nowrap; - } - /* Upload (paths) */ .upload-input { font-size: 11px; @@ -639,7 +628,7 @@
Roads
1
-
Download county data
+
Download & convert county data
@@ -651,15 +640,8 @@
-
-
3
-
Convert to GeoJSON
-
- -
-
-
4
+
3
Diff vs OSM
@@ -672,7 +654,7 @@
Paths
1
-
Upload county paths
+
Upload & convert county paths
@@ -684,17 +666,10 @@
-
-
3
-
Convert to GeoJSON
-
- -
-
-
4
+
3
Diff vs OSM
-
+
@@ -705,7 +680,7 @@
Addresses
1
-
Download county data
+
Download & convert county data
@@ -717,25 +692,15 @@
-
-
3
-
Convert to GeoJSON
-
- -
-
-
4
+
3
Diff vs OSM
-
-
5
+
4
Generate City Boundaries Report
@@ -840,7 +805,7 @@ const na = 'Not available'; if (!pathsUrl) { - ['1','2','3','4'].forEach(n => { + ['1','2','3'].forEach(n => { document.getElementById(`paths-step-${n}-actions`).innerHTML = na; }); return; @@ -858,8 +823,8 @@ ``; } - // Steps 2–4 - [['2','download-osm-paths'],['3','convert-paths'],['4','diff-paths']].forEach(([n, script]) => { + // Steps 2–3 + [['2','download-osm-paths'],['3','diff-paths']].forEach(([n, script]) => { document.getElementById(`paths-step-${n}-actions`).innerHTML = ``; }); @@ -958,20 +923,13 @@ document.querySelector('.right').scrollIntoView({ behavior: 'smooth', block: 'start' }); } - let forceDownload = false; - if (scriptName === 'diff-addresses') { - const cb = document.getElementById('force-redownload'); - if (cb) forceDownload = cb.checked; - } - fetch('/api/run-script', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ script: scriptName, county: scriptName === 'ls' ? '' : activeCounty, - label: jobLabel({ script: scriptName }), - forceDownload + label: jobLabel({ script: scriptName }) }) }) .then(r => r.json()) @@ -1009,6 +967,7 @@ /* ── Shared job status (same on every device) ── */ function jobLabel(job) { if (job.script === 'ls') return 'List files'; + if (job.script === 'convert-paths') return 'Paths · Convert uploaded county paths'; const step = document.querySelector(`.step[data-script="${job.script}"]`); if (!step) return job.script; const group = step.closest('.wf-group').querySelector('.wf-title').textContent.trim(); @@ -1298,6 +1257,7 @@ showStatus(`Uploading ${file.name}...`, 'info'); uploading = true; disableButtons(); + let convert = false; fetch('/api/upload-file', { method: 'POST', body: formData }) .then(r => r.json()) @@ -1308,10 +1268,16 @@ showStatus(`Uploaded to ${data.path}`, 'success'); refreshFiles(); input.value = ''; + if (dest.endsWith('/paths.shp.zip')) convert = true; } }) .catch(err => showStatus(`Error: ${err.message}`, 'error')) - .finally(() => { uploading = false; refreshJobs(); }); + .finally(() => { + uploading = false; + // Uploaded county paths still need converting to GeoJSON + if (convert) runScript('convert-paths'); + else refreshJobs(); + }); } function clearData() {