Final versions working for Lake and Sumter

README.md

@@ -7,6 +7,9 @@ See compare-addresses.py for an automated way of running the complete address di
 
 ## New Instructions
 
+* NOTE: when downloading OSM data towards the end via JOSM, copy-paste the output of the download script but add `(._;>;);out;` to the end instead of `out geom;` so JOSM picks it up.
+* NOTE: also add `way["highway"="construction"](area.searchArea);way["highway"="path"](area.searchArea);way["highway"="cycleway"](area.searchArea);` to the end so that roads under construction and cartpaths show up in JOSM to be analyzed/replaced/modified/etc.
+
 ### Roads
 
 * Get new data from the county and convert it: 
@@ -16,15 +19,20 @@ See compare-addresses.py for an automated way of running the complete address di
   * Sumter: `python download-overpass.py --type highways "Sumter County" "Florida" "original data/Sumter/osm-sumter-roads-$(date +%y%m%d).geojson"`
   * Lake: `python download-overpass.py --type highways "Lake County" "Florida" "original data/Lake/osm-lake-roads-$(date +%y%m%d).geojson"`
 * Diff the roads:
-  * Sumter (change 041125): `python threaded.py --output 'processed data\Sumter\diff-sumter-roads-$(date +%y%m%d).geojson' 'original data\Sumter\osm-sumter-roads-$(date +%y%m%d).geojson' 'original data\Sumter\RoadCenterlines_041125.geojson'`
-  * Lake (change 2025-06): `python threaded.py --output 'processed data\Lake\diff-lake-roads-$(date +%y%m%d).geojson' 'original data\Lake\osm-lake-roads-$(date +%y%m%d).geojson' 'original data\Lake\Streets 2025-06.geojson'`
+  * Sumter (change 041125): `python threaded.py --output "processed data\Sumter\diff-sumter-roads-$(date +%y%m%d).geojson" "original data\Sumter\osm-sumter-roads-$(date +%y%m%d).geojson" "original data\Sumter\RoadCenterlines_041125.geojson"`
+  * Lake (change 2025-06): `python threaded.py --output "processed data\Lake\diff-lake-roads-$(date +%y%m%d).geojson" "original data\Lake\osm-lake-roads-$(date +%y%m%d).geojson" "original data\Lake\Streets 2025-06.geojson"`
 
 ## Data
 
 - Lake County Streets and Address Points: https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/
+  - Alternately:
+    - Streets: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/73
+    - Addresses: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/16
+    - Highways: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/9
 - Sumter GIS Road Centerlines, Addresses, and Multi Modal Trails is via emailing their GIS team and accessing their Dropbox (https://www.dropbox.com/scl/fo/67nh5y8e42tr2kzdmmcg4/AAsF7Ay0MRUN-e_Ajlh5yWQ?rlkey=h6u606av0d2zkszk9lm3qijlt&e=1&st=7j7i94f8&dl=0)
   - Alternately, roads: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/9177e17c72d3433aa79630c7eda84add/about
   - Addresses: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/c75c5aac13a648968c5596b0665be28b/about
+- Marion (TODO)
 
 ## Instructions
 

download-overpass.py

@@ -14,17 +14,43 @@ TODO:
 import argparse
 import json
 import sys
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from pathlib import Path
 
 
+def get_county_area_id(county_name, state_name):
+    """Get OSM area ID for a county using Nominatim."""
+    search_query = f"{county_name}, {state_name}, USA"
+    url = f"https://nominatim.openstreetmap.org/search?q={urllib.parse.quote(search_query)}&format=json&limit=1&featuretype=county"
+
+    # Nominatim requires User-Agent header
+    req = urllib.request.Request(url, headers={'User-Agent': 'TheVillagesImport/1.0'})
+
+    try:
+        with urllib.request.urlopen(req) as response:
+            results = json.loads(response.read().decode("utf-8"))
+
+        if results and results[0].get('osm_type') == 'relation':
+            relation_id = int(results[0]['osm_id'])
+            area_id = relation_id + 3600000000
+            print(f"Found {county_name}, {state_name}: relation {relation_id} -> area {area_id}")
+            return area_id
+
+        raise ValueError(f"Could not find relation for {county_name}, {state_name}")
+    except urllib.error.HTTPError as e:
+        print(f"Nominatim HTTP Error {e.code}: {e.reason}", file=sys.stderr)
+        sys.exit(1)
+
+
 def build_overpass_query(county_name, state_name, data_type="highways"):
     """Build Overpass API query for specified data type in a county."""
+    area_id = get_county_area_id(county_name, state_name)
+
     base_query = f"""[out:json][timeout:60];
-area["name"="{state_name}"]["admin_level"="4"]->.state;
-area["name"="{county_name}"](area.state)->.searchArea;"""
+area(id:{area_id})->.searchArea;"""
     
     if data_type == "highways":
         selector = '('

threaded.py

@@ -370,31 +370,56 @@ class RoadComparator:
                         #uncovered_length >= min_length_deg and
                         # Include entire original road with all original metadata
 
-                        #
-                        # For Sumter County Roads
-                        #
                         properties = {
                             'surface': 'asphalt'
                         }
 
-                        for key, value in original_properties.items():
-                            if key == 'NAME':
-                                properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None
-                            elif key == 'SpeedLimit':
-                                properties['maxspeed'] = f"{value} mph" if value is not None else None
-                            elif key == 'RoadClass':
-                                if value is None:
-                                    properties['highway'] = 'residential'
-                                elif value.startswith('PRIMARY'):
-                                    properties['highway'] = 'trunk'
-                                elif value.startswith('MAJOR'):
-                                    properties['highway'] = 'primary'
-                                #elif value.startswith('MINOR'):
-                                else:
-                                    properties['highway'] = 'residential'
-                            # else:
-                            #     # Keep other properties as-is, or transform them as needed
-                            #     properties[key] = value
+                        # Detect county format based on available fields
+                        is_lake_county = 'FullStreet' in original_properties
+                        is_sumter_county = 'NAME' in original_properties and 'RoadClass' in original_properties
+
+                        if is_lake_county:
+                            # Lake County field mappings
+                            for key, value in original_properties.items():
+                                if key == 'FullStreet':
+                                    properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None
+                                elif key == 'SpeedLimit':
+                                    properties['maxspeed'] = f"{value} mph" if value is not None else None
+                                elif key == 'NumberOfLa':
+                                    try:
+                                        num_value = int(float(value)) if value is not None else 0
+                                        if num_value > 0:
+                                            properties['lanes'] = str(num_value)
+                                    except (ValueError, TypeError):
+                                        pass
+                                elif key == 'StreetClas':
+                                    highway_type = qgisfunctions.gethighwaytype(value, None, None)
+                                    properties['highway'] = highway_type if highway_type else 'residential'
+                        elif is_sumter_county:
+                            # Sumter County field mappings
+                            for key, value in original_properties.items():
+                                if key == 'NAME':
+                                    properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None
+                                elif key == 'SpeedLimit':
+                                    properties['maxspeed'] = f"{value} mph" if value is not None else None
+                                elif key == 'RoadClass':
+                                    if value is None:
+                                        properties['highway'] = 'residential'
+                                    elif value.startswith('PRIMARY'):
+                                        properties['highway'] = 'trunk'
+                                    elif value.startswith('MAJOR'):
+                                        properties['highway'] = 'primary'
+                                    else:
+                                        properties['highway'] = 'residential'
+                        else:
+                            # Unknown format - try common field names
+                            name = original_properties.get('NAME') or original_properties.get('FullStreet') or original_properties.get('name')
+                            if name:
+                                properties['name'] = titlecase(qgisfunctions.formatstreet(name,None,None))
+                            speed = original_properties.get('SpeedLimit')
+                            if speed:
+                                properties['maxspeed'] = f"{speed} mph"
+                            properties['highway'] = 'residential'
 
                         added_roads.append({
                             'geometry': geom,
@@ -461,9 +486,20 @@ class RoadComparator:
         print("Saving results...")
         results_gdf = gpd.GeoDataFrame(all_results)
         
-        # Save to file with optimization
-        results_gdf.to_file(output_path, driver='GeoJSON', engine='pyogrio')
-        print(f"Results saved to: {output_path}")
+        # Save to file with optimization, with fallback for locked files
+        try:
+            results_gdf.to_file(output_path, driver='GeoJSON', engine='pyogrio')
+            print(f"Results saved to: {output_path}")
+        except (PermissionError, OSError) as e:
+            # File is locked, try with a timestamp suffix
+            from datetime import datetime
+            timestamp = datetime.now().strftime("%H%M%S")
+            base = Path(output_path)
+            fallback_path = str(base.parent / f"{base.stem}_{timestamp}{base.suffix}")
+            print(f"Warning: Could not save to {output_path}: {e}")
+            print(f"Saving to fallback: {fallback_path}")
+            results_gdf.to_file(fallback_path, driver='GeoJSON', engine='pyogrio')
+            print(f"Results saved to: {fallback_path}")
     
     def print_summary(self, removed: List[Dict], added: List[Dict], file1_name: str, file2_name: str):
         """Print a summary of the comparison results."""