From da55bc29a6394da96720f1fcde67a4d0c5c650c3 Mon Sep 17 00:00:00 2001 From: Will Bradley Date: Wed, 26 Nov 2025 13:05:51 -0800 Subject: [PATCH] Final versions working for Lake and Sumter --- README.md | 12 +++++-- download-overpass.py | 30 ++++++++++++++-- threaded.py | 86 +++++++++++++++++++++++++++++++------------- 3 files changed, 99 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index d8fde72..fa230d2 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ See compare-addresses.py for an automated way of running the complete address di ## New Instructions +* NOTE: when downloading OSM data towards the end via JOSM, copy-paste the output of the download script but add `(._;>;);out;` to the end instead of `out geom;` so JOSM picks it up. +* NOTE: also add `way["highway"="construction"](area.searchArea);way["highway"="path"](area.searchArea);way["highway"="cycleway"](area.searchArea);` to the end so that roads under construction and cartpaths show up in JOSM to be analyzed/replaced/modified/etc. + ### Roads * Get new data from the county and convert it: @@ -16,15 +19,20 @@ See compare-addresses.py for an automated way of running the complete address di * Sumter: `python download-overpass.py --type highways "Sumter County" "Florida" "original data/Sumter/osm-sumter-roads-$(date +%y%m%d).geojson"` * Lake: `python download-overpass.py --type highways "Lake County" "Florida" "original data/Lake/osm-lake-roads-$(date +%y%m%d).geojson"` * Diff the roads: - * Sumter (change 041125): `python threaded.py --output 'processed data\Sumter\diff-sumter-roads-$(date +%y%m%d).geojson' 'original data\Sumter\osm-sumter-roads-$(date +%y%m%d).geojson' 'original data\Sumter\RoadCenterlines_041125.geojson'` - * Lake (change 2025-06): `python threaded.py --output 'processed data\Lake\diff-lake-roads-$(date +%y%m%d).geojson' 'original data\Lake\osm-lake-roads-$(date +%y%m%d).geojson' 'original data\Lake\Streets 2025-06.geojson'` + * Sumter (change 041125): `python threaded.py --output "processed data\Sumter\diff-sumter-roads-$(date +%y%m%d).geojson" "original data\Sumter\osm-sumter-roads-$(date +%y%m%d).geojson" "original data\Sumter\RoadCenterlines_041125.geojson"` + * Lake (change 2025-06): `python threaded.py --output "processed data\Lake\diff-lake-roads-$(date +%y%m%d).geojson" "original data\Lake\osm-lake-roads-$(date +%y%m%d).geojson" "original data\Lake\Streets 2025-06.geojson"` ## Data - Lake County Streets and Address Points: https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/ + - Alternately: + - Streets: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/73 + - Addresses: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/16 + - Highways: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/9 - Sumter GIS Road Centerlines, Addresses, and Multi Modal Trails is via emailing their GIS team and accessing their Dropbox (https://www.dropbox.com/scl/fo/67nh5y8e42tr2kzdmmcg4/AAsF7Ay0MRUN-e_Ajlh5yWQ?rlkey=h6u606av0d2zkszk9lm3qijlt&e=1&st=7j7i94f8&dl=0) - Alternately, roads: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/9177e17c72d3433aa79630c7eda84add/about - Addresses: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/c75c5aac13a648968c5596b0665be28b/about +- Marion (TODO) ## Instructions diff --git a/download-overpass.py b/download-overpass.py index 9620a4f..2e34bf0 100644 --- a/download-overpass.py +++ b/download-overpass.py @@ -14,17 +14,43 @@ TODO: import argparse import json import sys +import time import urllib.error import urllib.parse import urllib.request from pathlib import Path +def get_county_area_id(county_name, state_name): + """Get OSM area ID for a county using Nominatim.""" + search_query = f"{county_name}, {state_name}, USA" + url = f"https://nominatim.openstreetmap.org/search?q={urllib.parse.quote(search_query)}&format=json&limit=1&featuretype=county" + + # Nominatim requires User-Agent header + req = urllib.request.Request(url, headers={'User-Agent': 'TheVillagesImport/1.0'}) + + try: + with urllib.request.urlopen(req) as response: + results = json.loads(response.read().decode("utf-8")) + + if results and results[0].get('osm_type') == 'relation': + relation_id = int(results[0]['osm_id']) + area_id = relation_id + 3600000000 + print(f"Found {county_name}, {state_name}: relation {relation_id} -> area {area_id}") + return area_id + + raise ValueError(f"Could not find relation for {county_name}, {state_name}") + except urllib.error.HTTPError as e: + print(f"Nominatim HTTP Error {e.code}: {e.reason}", file=sys.stderr) + sys.exit(1) + + def build_overpass_query(county_name, state_name, data_type="highways"): """Build Overpass API query for specified data type in a county.""" + area_id = get_county_area_id(county_name, state_name) + base_query = f"""[out:json][timeout:60]; -area["name"="{state_name}"]["admin_level"="4"]->.state; -area["name"="{county_name}"](area.state)->.searchArea;""" +area(id:{area_id})->.searchArea;""" if data_type == "highways": selector = '(' diff --git a/threaded.py b/threaded.py index 5493e1e..b56a838 100644 --- a/threaded.py +++ b/threaded.py @@ -367,34 +367,59 @@ class RoadComparator: # 1. The uncovered portion is above minimum threshold, AND # 2. More than 10% of the road is uncovered if uncovered_ratio > 0.1: - #uncovered_length >= min_length_deg and + #uncovered_length >= min_length_deg and # Include entire original road with all original metadata - # - # For Sumter County Roads - # properties = { 'surface': 'asphalt' } - for key, value in original_properties.items(): - if key == 'NAME': - properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None - elif key == 'SpeedLimit': - properties['maxspeed'] = f"{value} mph" if value is not None else None - elif key == 'RoadClass': - if value is None: - properties['highway'] = 'residential' - elif value.startswith('PRIMARY'): - properties['highway'] = 'trunk' - elif value.startswith('MAJOR'): - properties['highway'] = 'primary' - #elif value.startswith('MINOR'): - else: - properties['highway'] = 'residential' - # else: - # # Keep other properties as-is, or transform them as needed - # properties[key] = value + # Detect county format based on available fields + is_lake_county = 'FullStreet' in original_properties + is_sumter_county = 'NAME' in original_properties and 'RoadClass' in original_properties + + if is_lake_county: + # Lake County field mappings + for key, value in original_properties.items(): + if key == 'FullStreet': + properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None + elif key == 'SpeedLimit': + properties['maxspeed'] = f"{value} mph" if value is not None else None + elif key == 'NumberOfLa': + try: + num_value = int(float(value)) if value is not None else 0 + if num_value > 0: + properties['lanes'] = str(num_value) + except (ValueError, TypeError): + pass + elif key == 'StreetClas': + highway_type = qgisfunctions.gethighwaytype(value, None, None) + properties['highway'] = highway_type if highway_type else 'residential' + elif is_sumter_county: + # Sumter County field mappings + for key, value in original_properties.items(): + if key == 'NAME': + properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None + elif key == 'SpeedLimit': + properties['maxspeed'] = f"{value} mph" if value is not None else None + elif key == 'RoadClass': + if value is None: + properties['highway'] = 'residential' + elif value.startswith('PRIMARY'): + properties['highway'] = 'trunk' + elif value.startswith('MAJOR'): + properties['highway'] = 'primary' + else: + properties['highway'] = 'residential' + else: + # Unknown format - try common field names + name = original_properties.get('NAME') or original_properties.get('FullStreet') or original_properties.get('name') + if name: + properties['name'] = titlecase(qgisfunctions.formatstreet(name,None,None)) + speed = original_properties.get('SpeedLimit') + if speed: + properties['maxspeed'] = f"{speed} mph" + properties['highway'] = 'residential' added_roads.append({ 'geometry': geom, @@ -461,9 +486,20 @@ class RoadComparator: print("Saving results...") results_gdf = gpd.GeoDataFrame(all_results) - # Save to file with optimization - results_gdf.to_file(output_path, driver='GeoJSON', engine='pyogrio') - print(f"Results saved to: {output_path}") + # Save to file with optimization, with fallback for locked files + try: + results_gdf.to_file(output_path, driver='GeoJSON', engine='pyogrio') + print(f"Results saved to: {output_path}") + except (PermissionError, OSError) as e: + # File is locked, try with a timestamp suffix + from datetime import datetime + timestamp = datetime.now().strftime("%H%M%S") + base = Path(output_path) + fallback_path = str(base.parent / f"{base.stem}_{timestamp}{base.suffix}") + print(f"Warning: Could not save to {output_path}: {e}") + print(f"Saving to fallback: {fallback_path}") + results_gdf.to_file(fallback_path, driver='GeoJSON', engine='pyogrio') + print(f"Results saved to: {fallback_path}") def print_summary(self, removed: List[Dict], added: List[Dict], file1_name: str, file2_name: str): """Print a summary of the comparison results."""