will/osm-import-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fold GeoJSON conversion into the county download steps and remove the unused Force redownload checkbox

Browse Source
This commit is contained in:
zyphlar
2026-10-22 11:31:34 -07:00
parent 4759424e85
commit 5bec6deb19
3 changed files with 36 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/server.py
+12 -12
View File
@@ -481,7 +481,7 @@ def get_script_map():
dl_roads = {}; dl_addrs = {}; dl_paths = {}
osm_roads = {}; osm_addrs = {}; osm_paths = {}
conv_roads = {}; conv_addrs = {}; conv_paths = {}
conv_paths = {}
diff_roads = {}; diff_addrs = {}; diff_paths = {}
city_report = {}
@@ -491,26 +491,29 @@ def get_script_map():
state = c['state']
base = f'/data/{cid}'
# Each county download also converts the shapefile to GeoJSON, so the
# diff can run straight after it. Uploaded paths are converted by a
# separate convert-paths job that the dashboard starts after the upload.
conv_roads = f'python shp-to-geojson.py {base}/roads.shp.zip {base}/county-roads.geojson'
conv_addrs = f'python convert-addresses.py {base}/addresses.shp.zip {base}/county-addresses.geojson'
conv_paths_cmd = f'python shp-to-geojson.py {base}/paths.shp.zip {base}/county-paths.geojson'
if c.get('roads_url'):
dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip'
dl_roads[cid] = f'mkdir -p {base} && wget {c["roads_url"]} -O {base}/roads.shp.zip && {conv_roads}'
if c.get('addresses_url'):
dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip'
dl_addrs[cid] = f'mkdir -p {base} && wget {c["addresses_url"]} -O {base}/addresses.shp.zip && {conv_addrs}'
pu = c.get('paths_url', '')
if pu == 'upload':
dl_paths[cid] = {'type': 'upload', 'dest': f'{base}/paths.shp.zip'}
conv_paths[cid] = conv_paths_cmd
elif pu:
dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip'
dl_paths[cid] = f'mkdir -p {base} && wget {pu} -O {base}/paths.shp.zip && {conv_paths_cmd}'
osm_roads[cid] = ['python', 'download-overpass.py', '--type', 'highways', name, state, f'{base}/osm-roads.geojson']
osm_addrs[cid] = ['python', 'download-overpass.py', '--type', 'addresses', name, state, f'{base}/osm-addresses.geojson']
if pu:
osm_paths[cid] = ['python', 'download-overpass.py', '--type', 'paths', name, state, f'{base}/osm-paths.geojson']
conv_roads[cid] = ['python', 'shp-to-geojson.py', f'{base}/roads.shp.zip', f'{base}/county-roads.geojson']
conv_addrs[cid] = ['python', 'convert-addresses.py', f'{base}/addresses.shp.zip', f'{base}/county-addresses.geojson']
if pu:
conv_paths[cid] = ['python', 'shp-to-geojson.py', f'{base}/paths.shp.zip', f'{base}/county-paths.geojson']
diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson']
diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base]
city_report[cid] = ['python', 'generate-city-report.py', '--existing-file', f'{base}/addresses-existing.geojson', '--output-dir', base]
@@ -523,8 +526,6 @@ def get_script_map():
sm['download-osm-roads'] = osm_roads
sm['download-osm-addresses'] = osm_addrs
if osm_paths: sm['download-osm-paths'] = osm_paths
sm['convert-roads'] = conv_roads
sm['convert-addresses'] = conv_addrs
if conv_paths: sm['convert-paths'] = conv_paths
sm['diff-roads'] = diff_roads
sm['diff-addresses'] = diff_addrs
@@ -552,7 +553,6 @@ def run_script():
data = request.json
script_name = data.get('script')
county = data.get('county', '')
force_download = data.get('forceDownload', False)
if not script_name:
return jsonify({'error': 'No script specified'}), 400
web/templates/index.html
+21 -55
View File
@@ -215,17 +215,6 @@
.left .btn-run { background: var(--active-color, #28a745); }
/* Force redownload */
.force-label {
font-size: 10px;
color: #aaa;
display: flex;
align-items: center;
gap: 3px;
cursor: pointer;
white-space: nowrap;
}
/* Upload (paths) */
.upload-input {
font-size: 11px;
@@ -639,7 +628,7 @@
<div class="wf-title">Roads</div>
<div class="step" data-script="download-county-roads">
<div class="step-num">1</div>
<div class="step-label"><span class="step-name">Download county data</span><button type="button" class="step-last" hidden></button></div>
<div class="step-label"><span class="step-name">Download &amp; convert county data</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('download-county-roads')">Run</button>
</div>
@@ -651,15 +640,8 @@
<button class="btn-run" onclick="runScript('download-osm-roads')">Run</button>
</div>
</div>
<div class="step" data-script="convert-roads">
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Convert to GeoJSON</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('convert-roads')">Run</button>
</div>
</div>
<div class="step" data-script="diff-roads">
<div class="step-num">4</div>
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('diff-roads')">Run</button>
@@ -672,7 +654,7 @@
<div class="wf-title">Paths</div>
<div class="step" id="paths-step-1" data-script="download-county-paths">
<div class="step-num">1</div>
<div class="step-label"><span class="step-name">Upload county paths</span><button type="button" class="step-last" hidden></button></div>
<div class="step-label"><span class="step-name">Upload &amp; convert county paths</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions" id="paths-step-1-actions">
<!-- filled by selectCounty() -->
</div>
@@ -684,17 +666,10 @@
<!-- filled by selectCounty() -->
</div>
</div>
<div class="step" data-script="convert-paths">
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Convert to GeoJSON</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions" id="paths-step-3-actions">
<!-- filled by selectCounty() -->
</div>
</div>
<div class="step" data-script="diff-paths">
<div class="step-num">4</div>
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions" id="paths-step-4-actions">
<div class="step-actions" id="paths-step-3-actions">
<!-- filled by selectCounty() -->
</div>
</div>
@@ -705,7 +680,7 @@
<div class="wf-title">Addresses</div>
<div class="step" data-script="download-county-addresses">
<div class="step-num">1</div>
<div class="step-label"><span class="step-name">Download county data</span><button type="button" class="step-last" hidden></button></div>
<div class="step-label"><span class="step-name">Download &amp; convert county data</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('download-county-addresses')">Run</button>
</div>
@@ -717,25 +692,15 @@
<button class="btn-run" onclick="runScript('download-osm-addresses')">Run</button>
</div>
</div>
<div class="step" data-script="convert-addresses">
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Convert to GeoJSON</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('convert-addresses')">Run</button>
</div>
</div>
<div class="step" data-script="diff-addresses">
<div class="step-num">4</div>
<div class="step-num">3</div>
<div class="step-label"><span class="step-name">Diff vs OSM</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('diff-addresses')">Run</button>
<label class="force-label">
<input type="checkbox" id="force-redownload"> Force redownload
</label>
</div>
</div>
<div class="step" data-script="generate-city-report">
<div class="step-num">5</div>
<div class="step-num">4</div>
<div class="step-label"><span class="step-name">Generate City Boundaries Report</span><button type="button" class="step-last" hidden></button></div>
<div class="step-actions">
<button class="btn-run" onclick="runScript('generate-city-report')">Run</button>
@@ -840,7 +805,7 @@
const na = '<span class="na">Not available</span>';
if (!pathsUrl) {
['1','2','3','4'].forEach(n => {
['1','2','3'].forEach(n => {
document.getElementById(`paths-step-${n}-actions`).innerHTML = na;
});
return;
@@ -858,8 +823,8 @@
`<button class="btn-run" onclick="runScript('download-county-paths')">Run</button>`;
}
// Steps 2–4
[['2','download-osm-paths'],['3','convert-paths'],['4','diff-paths']].forEach(([n, script]) => {
// Steps 2–3
[['2','download-osm-paths'],['3','diff-paths']].forEach(([n, script]) => {
document.getElementById(`paths-step-${n}-actions`).innerHTML =
`<button class="btn-run" onclick="runScript('${script}')">Run</button>`;
});
@@ -958,20 +923,13 @@
document.querySelector('.right').scrollIntoView({ behavior: 'smooth', block: 'start' });
}
let forceDownload = false;
if (scriptName === 'diff-addresses') {
const cb = document.getElementById('force-redownload');
if (cb) forceDownload = cb.checked;
}
fetch('/api/run-script', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
script: scriptName,
county: scriptName === 'ls' ? '' : activeCounty,
label: jobLabel({ script: scriptName }),
forceDownload
label: jobLabel({ script: scriptName })
})
})
.then(r => r.json())
@@ -1009,6 +967,7 @@
/* ── Shared job status (same on every device) ── */
function jobLabel(job) {
if (job.script === 'ls') return 'List files';
if (job.script === 'convert-paths') return 'Paths · Convert uploaded county paths';
const step = document.querySelector(`.step[data-script="${job.script}"]`);
if (!step) return job.script;
const group = step.closest('.wf-group').querySelector('.wf-title').textContent.trim();
@@ -1298,6 +1257,7 @@
showStatus(`Uploading ${file.name}...`, 'info');
uploading = true;
disableButtons();
let convert = false;
fetch('/api/upload-file', { method: 'POST', body: formData })
.then(r => r.json())
@@ -1308,10 +1268,16 @@
showStatus(`Uploaded to ${data.path}`, 'success');
refreshFiles();
input.value = '';
if (dest.endsWith('/paths.shp.zip')) convert = true;
}
})
.catch(err => showStatus(`Error: ${err.message}`, 'error'))
.finally(() => { uploading = false; refreshJobs(); });
.finally(() => {
uploading = false;
// Uploaded county paths still need converting to GeoJSON
if (convert) runScript('convert-paths');
else refreshJobs();
});
}
function clearData() {