will/osm-import-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fold GeoJSON conversion into the county download steps and remove the unused Force redownload checkbox

Browse Source
This commit is contained in:
zyphlar
2026-10-22 11:31:34 -07:00
parent 4759424e85
commit 5bec6deb19
3 changed files with 36 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -4
View File
@@ -8,12 +8,11 @@
## Using ## Using
For each county, run the four steps in order using the dashboard. Each step must complete before the next: For each county, run the three steps in order using the dashboard. Each step must complete before the next:
1. **Download county data** — fetches the county dataset (or upload a shapefile ZIP for paths) 1. **Download & convert county data** — fetches the county dataset (or upload a shapefile ZIP for paths) and converts it to OSM-tagged GeoJSON
2. **Download OSM data** — queries Overpass for the relevant features in the county 2. **Download OSM data** — queries Overpass for the relevant features in the county
3. **Convert to GeoJSON** — converts the county data to OSM-tagged GeoJSON 3. **Diff vs OSM** — compares county data against OSM; produces added (green) and removed (red) GeoJSON
4. **Diff vs OSM** — compares county data against OSM; produces added (green) and removed (red) GeoJSON
Open the result in the Map Viewer. Green = features in county data missing from OSM (candidates to add). Red = features in OSM not in county data. Hide the OSM (gray) and county (purple) layers to focus on the diff. Click a feature to inspect its properties and accept or exclude it. Open accepted features in JOSM for final review before uploading. Open the result in the Map Viewer. Green = features in county data missing from OSM (candidates to add). Red = features in OSM not in county data. Hide the OSM (gray) and county (purple) layers to focus on the diff. Click a feature to inspect its properties and accept or exclude it. Open accepted features in JOSM for final review before uploading.
web/server.py
+12 -12
View File
@@ -481,7 +481,7 @@ def get_script_map():
dl_roads = {}; dl_addrs = {}; dl_paths = {} dl_roads = {}; dl_addrs = {}; dl_paths = {}
osm_roads = {}; osm_addrs = {}; osm_paths = {} osm_roads = {}; osm_addrs = {}; osm_paths = {}
conv_roads = {}; conv_addrs = {}; conv_paths = {} conv_paths = {}
diff_roads = {}; diff_addrs = {}; diff_paths = {} diff_roads = {}; diff_addrs = {}; diff_paths = {}
city_report = {} city_report = {}
@@ -491,26 +491,29 @@ def get_script_map():
state = c['state'] state = c['state']
base = f'/data/{cid}' base = f'/data/{cid}'
# Each county download also converts the shapefile to GeoJSON, so the
# diff can run straight after it. Uploaded paths are converted by a
# separate convert-paths job that the dashboard starts after the upload.
conv_roads = f'python shp-to-geojson.py {base}/roads.shp.zip {base}/county-roads.geojson'
conv_addrs = f'python convert-addresses.py {base}/addresses.shp.zip {base}/county-addresses.geojson'
conv_paths_cmd = f'python shp-to-geojson.py {base}/paths.shp.zip {base}/county-paths.geojson'
if c.get('roads_url'): if c.get('roads_url'):
dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip' dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip && {conv_roads}'
if c.get('addresses_url'): if c.get('addresses_url'):
dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip' dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip && {conv_addrs}'
pu = c.get('paths_url', '') pu = c.get('paths_url', '')
if pu == 'upload': if pu == 'upload':
dl_paths[cid] = {'type': 'upload', 'dest': f'{base}/paths.shp.zip'} dl_paths[cid] = {'type': 'upload', 'dest': f'{base}/paths.shp.zip'}
conv_paths[cid] = conv_paths_cmd
elif pu: elif pu:
dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip' dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip && {conv_paths_cmd}'
osm_roads[cid] = ['python', 'download-overpass.py', '--type', 'highways', name, state, f'{base}/osm-roads.geojson'] osm_roads[cid] = ['python', 'download-overpass.py', '--type', 'highways', name, state, f'{base}/osm-roads.geojson']
osm_addrs[cid] = ['python', 'download-overpass.py', '--type', 'addresses', name, state, f'{base}/osm-addresses.geojson'] osm_addrs[cid] = ['python', 'download-overpass.py', '--type', 'addresses', name, state, f'{base}/osm-addresses.geojson']
if pu: if pu:
osm_paths[cid] = ['python', 'download-overpass.py', '--type', 'paths', name, state, f'{base}/osm-paths.geojson'] osm_paths[cid] = ['python', 'download-overpass.py', '--type', 'paths', name, state, f'{base}/osm-paths.geojson']
conv_roads[cid] = ['python', 'shp-to-geojson.py', f'{base}/roads.shp.zip', f'{base}/county-roads.geojson']
conv_addrs[cid] = ['python', 'convert-addresses.py', f'{base}/addresses.shp.zip', f'{base}/county-addresses.geojson']
if pu:
conv_paths[cid] = ['python', 'shp-to-geojson.py', f'{base}/paths.shp.zip', f'{base}/county-paths.geojson']
diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson'] diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson']
diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base] diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base]
city_report[cid] = ['python', 'generate-city-report.py', '--existing-file', f'{base}/addresses-existing.geojson', '--output-dir', base] city_report[cid] = ['python', 'generate-city-report.py', '--existing-file', f'{base}/addresses-existing.geojson', '--output-dir', base]
@@ -523,8 +526,6 @@ def get_script_map():
sm['download-osm-roads'] = osm_roads sm['download-osm-roads'] = osm_roads
sm['download-osm-addresses'] = osm_addrs sm['download-osm-addresses'] = osm_addrs
if osm_paths: sm['download-osm-paths'] = osm_paths if osm_paths: sm['download-osm-paths'] = osm_paths
sm['convert-roads'] = conv_roads
sm['convert-addresses'] = conv_addrs
if conv_paths: sm['convert-paths'] = conv_paths if conv_paths: sm['convert-paths'] = conv_paths
sm['diff-roads'] = diff_roads sm['diff-roads'] = diff_roads
sm['diff-addresses'] = diff_addrs sm['diff-addresses'] = diff_addrs
@@ -552,7 +553,6 @@ def run_script():
data = request.json data = request.json
script_name = data.get('script') script_name = data.get('script')
county = data.get('county', '') county = data.get('county', '')
force_download = data.get('forceDownload', False)
if not script_name: if not script_name:
return jsonify({'error': 'No script specified'}), 400 return jsonify({'error': 'No script specified'}), 400
web/templates/index.html
+21 -55
View File
@@ -215,17 +215,6 @@
.left .btn-run { background: var(--active-color, #28a745); } .left .btn-run { background: var(--active-color, #28a745); }
/* Force redownload */
.force-label {
font-size: 10px;
color: #aaa;
display: flex;
align-items: center;
gap: 3px;
cursor: pointer;
white-space: nowrap;
}
/* Upload (paths) */ /* Upload (paths) */
.upload-input { .upload-input {
font-size: 11px; font-size: 11px;
@@ -639,7 +628,7 @@
<div class="wf-title">Roads</div> <div class="wf-title">Roads</div>
<div class="step" data-script="download-county-roads"> <div class="step" data-script="download-county-roads">
<div class="step-num">1</div> <div class="step-num">1</div>
<div class="step-label"><span class="step-name">Download county data</span><button type="button" class="step-last" hidden></button></div> <div class="step-label"><span class="step-name">Download &amp; convert county data</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions"> <div class="step-actions">
<button class="btn-run" onclick="runScript('download-county-roads')">Run</button> <button class="btn-run" onclick="runScript('download-county-roads')">Run</button>
</div> </div>
@@ -651,15 +640,8 @@
<button class="btn-run" onclick="runScript('download-osm-roads')">Run</button> <button class="btn-run" onclick="runScript('download-osm-roads')">Run</button>
</div> </div>
</div> </div>
<div class="step" data-script="convert-roads">
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Convert to GeoJSON</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('convert-roads')">Run</button>
</div>
</div>
<div class="step" data-script="diff-roads"> <div class="step" data-script="diff-roads">
<div class="step-num">4</div> <div class="step-num">3</div>
<div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div> <div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions"> <div class="step-actions">
<button class="btn-run" onclick="runScript('diff-roads')">Run</button> <button class="btn-run" onclick="runScript('diff-roads')">Run</button>
@@ -672,7 +654,7 @@
<div class="wf-title">Paths</div> <div class="wf-title">Paths</div>
<div class="step" id="paths-step-1" data-script="download-county-paths"> <div class="step" id="paths-step-1" data-script="download-county-paths">
<div class="step-num">1</div> <div class="step-num">1</div>
<div class="step-label"><span class="step-name">Upload county paths</span><button type="button" class="step-last" hidden></button></div> <div class="step-label"><span class="step-name">Upload &amp; convert county paths</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions" id="paths-step-1-actions"> <div class="step-actions" id="paths-step-1-actions">
<!-- filled by selectCounty() --> <!-- filled by selectCounty() -->
</div> </div>
@@ -684,17 +666,10 @@
<!-- filled by selectCounty() --> <!-- filled by selectCounty() -->
</div> </div>
</div> </div>
<div class="step" data-script="convert-paths">
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Convert to GeoJSON</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions" id="paths-step-3-actions">
<!-- filled by selectCounty() -->
</div>
</div>
<div class="step" data-script="diff-paths"> <div class="step" data-script="diff-paths">
<div class="step-num">4</div> <div class="step-num">3</div>
<div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div> <div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions" id="paths-step-4-actions"> <div class="step-actions" id="paths-step-3-actions">
<!-- filled by selectCounty() --> <!-- filled by selectCounty() -->
</div> </div>
</div> </div>
@@ -705,7 +680,7 @@
<div class="wf-title">Addresses</div> <div class="wf-title">Addresses</div>
<div class="step" data-script="download-county-addresses"> <div class="step" data-script="download-county-addresses">
<div class="step-num">1</div> <div class="step-num">1</div>
<div class="step-label"><span class="step-name">Download county data</span><button type="button" class="step-last" hidden></button></div> <div class="step-label"><span class="step-name">Download &amp; convert county data</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions"> <div class="step-actions">
<button class="btn-run" onclick="runScript('download-county-addresses')">Run</button> <button class="btn-run" onclick="runScript('download-county-addresses')">Run</button>
</div> </div>
@@ -717,25 +692,15 @@
<button class="btn-run" onclick="runScript('download-osm-addresses')">Run</button> <button class="btn-run" onclick="runScript('download-osm-addresses')">Run</button>
</div> </div>
</div> </div>
<div class="step" data-script="convert-addresses">
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Convert to GeoJSON</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('convert-addresses')">Run</button>
</div>
</div>
<div class="step" data-script="diff-addresses"> <div class="step" data-script="diff-addresses">
<div class="step-num">4</div> <div class="step-num">3</div>
<div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div> <div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions"> <div class="step-actions">
<button class="btn-run" onclick="runScript('diff-addresses')">Run</button> <button class="btn-run" onclick="runScript('diff-addresses')">Run</button>
<label class="force-label">
<input type="checkbox" id="force-redownload"> Force redownload
</label>
</div> </div>
</div> </div>
<div class="step" data-script="generate-city-report"> <div class="step" data-script="generate-city-report">
<div class="step-num">5</div> <div class="step-num">4</div>
<div class="step-label"><span class="step-name">Generate City Boundaries Report</span><button type="button" class="step-last" hidden></button></div> <div class="step-label"><span class="step-name">Generate City Boundaries Report</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions"> <div class="step-actions">
<button class="btn-run" onclick="runScript('generate-city-report')">Run</button> <button class="btn-run" onclick="runScript('generate-city-report')">Run</button>
@@ -840,7 +805,7 @@
const na = '<span class="na">Not available</span>'; const na = '<span class="na">Not available</span>';
if (!pathsUrl) { if (!pathsUrl) {
['1','2','3','4'].forEach(n => { ['1','2','3'].forEach(n => {
document.getElementById(`paths-step-${n}-actions`).innerHTML = na; document.getElementById(`paths-step-${n}-actions`).innerHTML = na;
}); });
return; return;
@@ -858,8 +823,8 @@
`<button class="btn-run" onclick="runScript('download-county-paths')">Run</button>`; `<button class="btn-run" onclick="runScript('download-county-paths')">Run</button>`;
} }
// Steps 2–4 // Steps 2–3
[['2','download-osm-paths'],['3','convert-paths'],['4','diff-paths']].forEach(([n, script]) => { [['2','download-osm-paths'],['3','diff-paths']].forEach(([n, script]) => {
document.getElementById(`paths-step-${n}-actions`).innerHTML = document.getElementById(`paths-step-${n}-actions`).innerHTML =
`<button class="btn-run" onclick="runScript('${script}')">Run</button>`; `<button class="btn-run" onclick="runScript('${script}')">Run</button>`;
}); });
@@ -958,20 +923,13 @@
document.querySelector('.right').scrollIntoView({ behavior: 'smooth', block: 'start' }); document.querySelector('.right').scrollIntoView({ behavior: 'smooth', block: 'start' });
} }
let forceDownload = false;
if (scriptName === 'diff-addresses') {
const cb = document.getElementById('force-redownload');
if (cb) forceDownload = cb.checked;
}
fetch('/api/run-script', { fetch('/api/run-script', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: JSON.stringify({
script: scriptName, script: scriptName,
county: scriptName === 'ls' ? '' : activeCounty, county: scriptName === 'ls' ? '' : activeCounty,
label: jobLabel({ script: scriptName }), label: jobLabel({ script: scriptName })
forceDownload
}) })
}) })
.then(r => r.json()) .then(r => r.json())
@@ -1009,6 +967,7 @@
/* ── Shared job status (same on every device) ── */ /* ── Shared job status (same on every device) ── */
function jobLabel(job) { function jobLabel(job) {
if (job.script === 'ls') return 'List files'; if (job.script === 'ls') return 'List files';
if (job.script === 'convert-paths') return 'Paths · Convert uploaded county paths';
const step = document.querySelector(`.step[data-script="${job.script}"]`); const step = document.querySelector(`.step[data-script="${job.script}"]`);
if (!step) return job.script; if (!step) return job.script;
const group = step.closest('.wf-group').querySelector('.wf-title').textContent.trim(); const group = step.closest('.wf-group').querySelector('.wf-title').textContent.trim();
@@ -1298,6 +1257,7 @@
showStatus(`Uploading ${file.name}...`, 'info'); showStatus(`Uploading ${file.name}...`, 'info');
uploading = true; uploading = true;
disableButtons(); disableButtons();
let convert = false;
fetch('/api/upload-file', { method: 'POST', body: formData }) fetch('/api/upload-file', { method: 'POST', body: formData })
.then(r => r.json()) .then(r => r.json())
@@ -1308,10 +1268,16 @@
showStatus(`Uploaded to ${data.path}`, 'success'); showStatus(`Uploaded to ${data.path}`, 'success');
refreshFiles(); refreshFiles();
input.value = ''; input.value = '';
if (dest.endsWith('/paths.shp.zip')) convert = true;
} }
}) })
.catch(err => showStatus(`Error: ${err.message}`, 'error')) .catch(err => showStatus(`Error: ${err.message}`, 'error'))
.finally(() => { uploading = false; refreshJobs(); }); .finally(() => {
uploading = false;
// Uploaded county paths still need converting to GeoJSON
if (convert) runScript('convert-paths');
else refreshJobs();
});
} }
function clearData() { function clearData() {