Merge branch 'ok-but-dupe-highways'

.gitignore

@@ -1,2 +1,5 @@
 __pycache__
-desktop.ini
+desktop.ini
+*.geojson
+osm_cache/
+.claude

README.md

@@ -2,10 +2,37 @@
 
 See [https://wiki.openstreetmap.org/wiki/The_Villages_Road_and_Address_Import](https://wiki.openstreetmap.org/wiki/The_Villages_Road_and_Address_Import)
 
+See compare-addresses.py for an automated way of running the complete address diff toolchain in one step.
+- TODO: fails to split out units
+
+## New Instructions
+
+* NOTE: when downloading OSM data towards the end via JOSM, copy-paste the output of the download script but add `(._;>;);out;` to the end instead of `out geom;` so JOSM picks it up.
+* NOTE: also add `way["highway"="construction"](area.searchArea);way["highway"="path"](area.searchArea);way["highway"="cycleway"](area.searchArea);` to the end so that roads under construction and cartpaths show up in JOSM to be analyzed/replaced/modified/etc.
+
+### Roads
+
+* Get new data from the county and convert it: 
+  * Sumter (change 041125): `python shp-to-geojson.py "original data/Sumter/RoadCenterlines_041125.shp.zip" "original data/Sumter/RoadCenterlines_041125.geojson"`
+  * Lake (change 2025-06): `python shp-to-geojson.py "original data/Lake/Streets 2025-06.zip" "original data/Lake/Streets 2025-06.geojson"`
+* Get new data from OSM:
+  * Sumter: `python download-overpass.py --type highways "Sumter County" "Florida" "original data/Sumter/osm-sumter-roads-$(date +%y%m%d).geojson"`
+  * Lake: `python download-overpass.py --type highways "Lake County" "Florida" "original data/Lake/osm-lake-roads-$(date +%y%m%d).geojson"`
+* Diff the roads:
+  * Sumter (change 041125): `python threaded.py --output "processed data\Sumter\diff-sumter-roads-$(date +%y%m%d).geojson" "original data\Sumter\osm-sumter-roads-$(date +%y%m%d).geojson" "original data\Sumter\RoadCenterlines_041125.geojson"`
+  * Lake (change 2025-06): `python threaded.py --output "processed data\Lake\diff-lake-roads-$(date +%y%m%d).geojson" "original data\Lake\osm-lake-roads-$(date +%y%m%d).geojson" "original data\Lake\Streets 2025-06.geojson"`
+
 ## Data
 
 - Lake County Streets and Address Points: https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/
+  - Alternately:
+    - Streets: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/73
+    - Addresses: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/16
+    - Highways: https://gis.lakecountyfl.gov/lakegis/rest/services/InteractiveMap/MapServer/9
 - Sumter GIS Road Centerlines, Addresses, and Multi Modal Trails is via emailing their GIS team and accessing their Dropbox (https://www.dropbox.com/scl/fo/67nh5y8e42tr2kzdmmcg4/AAsF7Ay0MRUN-e_Ajlh5yWQ?rlkey=h6u606av0d2zkszk9lm3qijlt&e=1&st=7j7i94f8&dl=0)
+  - Alternately, roads: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/9177e17c72d3433aa79630c7eda84add/about
+  - Addresses: https://test-sumter-county-open-data-sumtercountygis.hub.arcgis.com/datasets/c75c5aac13a648968c5596b0665be28b/about
+- Marion (TODO)
 
 ## Instructions
 
@@ -120,3 +147,22 @@ source:url=https://gitlab.com/zyphlar/the-villages-import
 
 
 * Review imported data in Achavi or Osmcha to ensure it looks proper.
+
+
+## Useful queries:
+
+```
+[timeout:60];
+area["name"="Florida"]->.state;
+area["name"="Lake County"](area.state)->.searchArea;nwr["addr:housenumber"](area.searchArea);
+(._;>;);
+out meta;
+```
+
+```
+[timeout:60];
+area["name"="Florida"]->.state;
+area["name"="Lake County"](area.state)->.searchArea;way["highway"](area.searchArea);
+(._;>;);
+out meta;
+```

compare-addresses.py

@@ -0,0 +1,627 @@
+#!/usr/bin/env python3
+"""
+Address Data Comparison Tool for US Counties
+
+Compares local government address data (from ZIP/shapefile) with OpenStreetMap address data.
+Downloads OSM data via Overpass API, converts local data to GeoJSON, and performs comprehensive
+comparison to identify new, existing, and removed addresses.
+
+Usage:
+    python compare-addresses.py "Lake" "Florida" --local-zip "original data/Lake/Addresspoints 2025-06.zip"
+    python compare-addresses.py "Sumter" "Florida" --local-zip "original data/Sumter/Address9_13_2024.zip"
+"""
+
+import argparse
+import json
+import os
+import sys
+import zipfile
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Tuple, Any, Optional
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import geopandas as gpd
+import pandas as pd
+from shapely.geometry import Point
+from shapely.strtree import STRtree
+from shapely.ops import nearest_points
+import warnings
+
+# Import local modules
+import importlib
+qgis_functions = importlib.import_module("qgis-functions")
+
+# Suppress warnings for cleaner output
+warnings.filterwarnings('ignore')
+
+
+class AddressComparator:
+    def __init__(self, tolerance_meters: float = 50.0, cache_dir: str = "osm_cache"):
+        """
+        Initialize the address comparator.
+        
+        Args:
+            tolerance_meters: Distance tolerance for considering addresses as matching
+            cache_dir: Directory to cache OSM data
+        """
+        self.tolerance_meters = tolerance_meters
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(exist_ok=True)
+        
+        # Convert meters to degrees (approximate)
+        # 1 degree latitude ≈ 111,000 meters
+        self.tolerance_deg = tolerance_meters / 111000.0
+        
+    def download_osm_addresses(self, county: str, state: str, output_file: str = None) -> str:
+        """Download address data from OpenStreetMap via Overpass API."""
+        if output_file is None:
+            timestamp = datetime.now().strftime("%Y%m%d")
+            output_file = self.cache_dir / f"osm_addresses_{county.lower()}_{timestamp}.geojson"
+        else:
+            output_file = Path(output_file)
+            
+        # Check if cached file exists and is recent (less than 7 days old)
+        if output_file.exists():
+            file_age = datetime.now().timestamp() - output_file.stat().st_mtime
+            if file_age < 7 * 24 * 3600:  # 7 days in seconds
+                print(f"Using cached OSM data: {output_file}")
+                return str(output_file)
+        
+        print(f"Downloading OSM addresses for {county} County, {state}...")
+        
+        # Build Overpass query for addresses
+        query = f"""[out:json][timeout:180];
+area["name"="{state}"]->.state;
+area["name"="{county} County"](area.state)->.searchArea;
+nwr["addr:housenumber"](area.searchArea);
+out geom;"""
+        
+        # Query Overpass API
+        osm_data = self._query_overpass(query)
+        
+        # Convert to GeoJSON
+        geojson = self._convert_osm_to_geojson(osm_data)
+        
+        # Save to file
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        with open(output_file, 'w', encoding='utf-8') as f:
+            json.dump(geojson, f, indent=2)
+        
+        print(f"Downloaded {len(geojson['features'])} OSM addresses to {output_file}")
+        return str(output_file)
+    
+    def _query_overpass(self, query: str) -> Dict[str, Any]:
+        """Send query to Overpass API and return JSON response."""
+        url = "https://overpass-api.de/api/interpreter"
+        data = urllib.parse.urlencode({"data": query}).encode("utf-8")
+        
+        try:
+            with urllib.request.urlopen(url, data=data, timeout=300) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as e:
+            print(f"HTTP Error {e.code}: {e.reason}", file=sys.stderr)
+            try:
+                error_body = e.read().decode("utf-8")
+                print(f"Error response: {error_body}", file=sys.stderr)
+            except:
+                pass
+            sys.exit(1)
+        except Exception as e:
+            print(f"Error querying Overpass API: {e}", file=sys.stderr)
+            sys.exit(1)
+    
+    def _convert_osm_to_geojson(self, overpass_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert Overpass API response to GeoJSON format."""
+        features = []
+        
+        for element in overpass_data.get("elements", []):
+            properties = element.get("tags", {})
+            
+            # Extract coordinates based on element type
+            if element["type"] == "node":
+                coordinates = [element["lon"], element["lat"]]
+                geometry = {"type": "Point", "coordinates": coordinates}
+            elif element["type"] == "way" and "geometry" in element:
+                # For ways, use the centroid
+                coords = [[coord["lon"], coord["lat"]] for coord in element["geometry"]]
+                if len(coords) > 0:
+                    # Calculate centroid
+                    lon = sum(coord[0] for coord in coords) / len(coords)
+                    lat = sum(coord[1] for coord in coords) / len(coords)
+                    coordinates = [lon, lat]
+                    geometry = {"type": "Point", "coordinates": coordinates}
+                else:
+                    continue
+            else:
+                continue  # Skip relations and ways without geometry
+            
+            feature = {
+                "type": "Feature",
+                "properties": properties,
+                "geometry": geometry
+            }
+            features.append(feature)
+        
+        return {
+            "type": "FeatureCollection",
+            "features": features
+        }
+    
+    def load_local_addresses(self, zip_path: str, output_geojson: str = None) -> str:
+        """Load and convert local address data from ZIP file."""
+        zip_path = Path(zip_path)
+        
+        if output_geojson is None:
+            output_geojson = zip_path.parent / f"{zip_path.stem}_converted.geojson"
+        else:
+            output_geojson = Path(output_geojson)
+        
+        # Check if conversion already exists and is newer than the ZIP
+        if (output_geojson.exists() and 
+            zip_path.exists() and 
+            output_geojson.stat().st_mtime > zip_path.stat().st_mtime):
+            print(f"Using existing converted data: {output_geojson}")
+            return str(output_geojson)
+        
+        print(f"Converting local address data from {zip_path}...")
+        
+        # Extract and find shapefile in ZIP
+        temp_dir = zip_path.parent / "temp_extract"
+        temp_dir.mkdir(exist_ok=True)
+        
+        try:
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                zip_ref.extractall(temp_dir)
+            
+            # Find the shapefile
+            shp_files = list(temp_dir.glob("*.shp"))
+            if not shp_files:
+                raise FileNotFoundError("No shapefile (.shp) found in ZIP")
+            
+            shp_file = shp_files[0]
+            
+            # Load and process shapefile
+            gdf = gpd.read_file(shp_file)
+            
+            # Convert CRS to WGS84 if needed
+            if gdf.crs and gdf.crs != 'EPSG:4326':
+                print(f"Converting from {gdf.crs} to EPSG:4326")
+                gdf = gdf.to_crs('EPSG:4326')
+            
+            # Process address fields using existing logic from sumter-address-convert.py
+            gdf = self._process_address_fields(gdf)
+            
+            # Filter to only point geometries and valid addresses
+            gdf = gdf[gdf.geometry.type == 'Point'].copy()
+            gdf = gdf[gdf['addr:housenumber'].notna()].copy()
+            
+            # Clean output data - keep only OSM address fields
+            osm_fields = [
+                'addr:housenumber', 'addr:unit', 'addr:street', 
+                'addr:city', 'addr:postcode', 'addr:state'
+            ]
+            existing_fields = [field for field in osm_fields if field in gdf.columns]
+            gdf = gdf[existing_fields + ['geometry']]
+            
+            # Save to GeoJSON
+            output_geojson.parent.mkdir(parents=True, exist_ok=True)
+            gdf.to_file(output_geojson, driver='GeoJSON')
+            
+            print(f"Converted {len(gdf)} addresses to {output_geojson}")
+            
+        finally:
+            # Clean up temp directory
+            import shutil
+            if temp_dir.exists():
+                shutil.rmtree(temp_dir)
+        
+        return str(output_geojson)
+    
+    def _process_address_fields(self, gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
+        """Process address fields according to OSM schema (handles multiple formats)."""
+        processed_gdf = gdf.copy()
+        address_mapping = {}
+        
+        # House number - try multiple field names
+        house_number_fields = ['ADD_NUM', 'AddressNum', 'ADDRESS_NUM', 'HOUSE_NUM']
+        for field in house_number_fields:
+            if field in processed_gdf.columns:
+                add_num_series = processed_gdf[field].copy()
+                add_num_series = pd.to_numeric(add_num_series, errors='coerce')
+                address_mapping['addr:housenumber'] = add_num_series.round().astype('Int64')
+                break
+        
+        # Unit number - try multiple field names
+        unit_fields = ['UNIT', 'UnitNumber', 'UNIT_NUM', 'APT']
+        for field in unit_fields:
+            if field in processed_gdf.columns:
+                unit_series = processed_gdf[field].copy()
+                unit_series = unit_series.replace(['nan', 'None', '', None], None)
+                unit_series = unit_series.where(unit_series.notna(), None)
+                address_mapping['addr:unit'] = unit_series
+                break
+        
+        # Street name - try multiple approaches
+        if 'SADD' in processed_gdf.columns:
+            # Sumter County format - full address in SADD field
+            street_names = []
+            for sadd_value in processed_gdf['SADD']:
+                if pd.notna(sadd_value):
+                    street_from_addr = qgis_functions.getstreetfromaddress(str(sadd_value), None, None)
+                    street_titled = qgis_functions.title(street_from_addr)
+                    street_names.append(street_titled)
+                else:
+                    street_names.append(None)
+            address_mapping['addr:street'] = street_names
+        elif 'FullAddres' in processed_gdf.columns:
+            # Lake County format - full address in FullAddres field
+            street_names = []
+            for full_addr in processed_gdf['FullAddres']:
+                if pd.notna(full_addr):
+                    street_from_addr = qgis_functions.getstreetfromaddress(str(full_addr), None, None)
+                    street_titled = qgis_functions.title(street_from_addr)
+                    street_names.append(street_titled)
+                else:
+                    street_names.append(None)
+            address_mapping['addr:street'] = street_names
+        elif 'BaseStreet' in processed_gdf.columns:
+            # Lake County alternative - combine street components
+            street_names = []
+            for idx, row in processed_gdf.iterrows():
+                street_parts = []
+                
+                # Prefix direction
+                if 'PrefixDire' in row and pd.notna(row['PrefixDire']):
+                    street_parts.append(str(row['PrefixDire']).strip())
+                
+                # Prefix type
+                if 'PrefixType' in row and pd.notna(row['PrefixType']):
+                    street_parts.append(str(row['PrefixType']).strip())
+                
+                # Base street name
+                if pd.notna(row['BaseStreet']):
+                    street_parts.append(str(row['BaseStreet']).strip())
+                
+                # Suffix type
+                if 'SuffixType' in row and pd.notna(row['SuffixType']):
+                    street_parts.append(str(row['SuffixType']).strip())
+                
+                if street_parts:
+                    street_name = ' '.join(street_parts)
+                    street_titled = qgis_functions.title(street_name)
+                    street_names.append(street_titled)
+                else:
+                    street_names.append(None)
+            
+            address_mapping['addr:street'] = street_names
+        
+        # City - try multiple field names
+        city_fields = ['POST_COMM', 'PostalCity', 'CITY', 'Jurisdicti']
+        for field in city_fields:
+            if field in processed_gdf.columns:
+                city_names = []
+                for city_value in processed_gdf[field]:
+                    if pd.notna(city_value):
+                        city_titled = qgis_functions.title(str(city_value))
+                        city_names.append(city_titled)
+                    else:
+                        city_names.append(None)
+                address_mapping['addr:city'] = city_names
+                break
+        
+        # Postal code - try multiple field names
+        postcode_fields = ['POST_CODE', 'ZipCode', 'ZIP', 'POSTAL_CODE']
+        for field in postcode_fields:
+            if field in processed_gdf.columns:
+                post_code_series = processed_gdf[field].copy()
+                post_code_series = pd.to_numeric(post_code_series, errors='coerce')
+                address_mapping['addr:postcode'] = post_code_series.round().astype('Int64')
+                break
+        
+        # Manually add addr:state
+        address_mapping['addr:state'] = 'FL'
+        
+        # Add the new address columns to the GeoDataFrame
+        for key, value in address_mapping.items():
+            processed_gdf[key] = value
+        
+        return processed_gdf
+    
+    def compare_addresses(self, local_file: str, osm_file: str) -> Tuple[List[Dict], List[Dict], List[Dict]]:
+        """
+        Compare local and OSM address data.
+        
+        Returns:
+            Tuple of (new_addresses, existing_addresses, removed_addresses)
+        """
+        print(f"Comparing addresses: {local_file} vs {osm_file}")
+        
+        # Load data
+        local_gdf = gpd.read_file(local_file)
+        osm_gdf = gpd.read_file(osm_file)
+        
+        print(f"Loaded local addresses: {len(local_gdf)}")
+        print(f"Loaded OSM addresses: {len(osm_gdf)}")
+        
+        # Apply sampling if requested
+        if hasattr(self, 'sample_size') and self.sample_size:
+            if len(local_gdf) > self.sample_size:
+                local_gdf = local_gdf.sample(n=self.sample_size, random_state=42).reset_index(drop=True)
+                print(f"Sampled local addresses to: {len(local_gdf)}")
+        
+        if hasattr(self, 'max_osm') and self.max_osm:
+            if len(osm_gdf) > self.max_osm:
+                osm_gdf = osm_gdf.sample(n=self.max_osm, random_state=42).reset_index(drop=True)
+                print(f"Sampled OSM addresses to: {len(osm_gdf)}")
+        
+        print(f"Processing local addresses: {len(local_gdf)}")
+        print(f"Processing OSM addresses: {len(osm_gdf)}")
+        
+        # Ensure both are in the same CRS
+        if local_gdf.crs != osm_gdf.crs:
+            osm_gdf = osm_gdf.to_crs(local_gdf.crs)
+        
+        # Create spatial indexes
+        local_index = STRtree(local_gdf.geometry.tolist())
+        osm_index = STRtree(osm_gdf.geometry.tolist())
+        
+        # Find matches
+        existing_addresses = []
+        new_addresses = []
+        
+        # For each local address, find closest OSM address
+        for idx, local_row in local_gdf.iterrows():
+            local_point = local_row.geometry
+            
+            # Query nearby OSM addresses
+            nearby_indices = osm_index.query(local_point.buffer(self.tolerance_deg))
+            
+            best_match = None
+            min_distance = float('inf')
+            
+            for osm_idx in nearby_indices:
+                osm_row = osm_gdf.iloc[osm_idx]
+                osm_point = osm_row.geometry
+                distance = local_point.distance(osm_point)
+                
+                # Additional verification: check if house numbers match (handle type differences)
+                local_house_num = str(local_row.get('addr:housenumber', ''))
+                osm_house_num = str(osm_row.get('addr:housenumber', ''))
+                
+                # Only consider as potential match if house numbers match
+                if local_house_num == osm_house_num and distance < min_distance:
+                    min_distance = distance
+                    best_match = osm_idx
+            
+            # Convert distance to meters for comparison
+            distance_meters = min_distance * 111000.0
+            
+            if best_match is not None and distance_meters <= self.tolerance_meters:
+                # Found a match - this is an existing address
+                local_props = dict(local_row.drop('geometry'))
+                osm_props = dict(osm_gdf.iloc[best_match].drop('geometry'))
+                
+                existing_addresses.append({
+                    'geometry': local_point,
+                    'local_data': local_props,
+                    'osm_data': osm_props,
+                    'distance_meters': distance_meters
+                })
+            else:
+                # No match found - this is a new address to add to OSM
+                local_props = dict(local_row.drop('geometry'))
+                local_props['status'] = 'new'
+                new_addresses.append({
+                    'geometry': local_point,
+                    **local_props
+                })
+        
+        # Find OSM addresses that don't have local matches (potentially removed)
+        removed_addresses = []
+        
+        # Track which OSM addresses were matched during the first pass
+        # by storing their index during the matching process above
+        matched_osm_indices = set()
+        
+        # Re-do the matching to track OSM indices
+        for idx, local_row in local_gdf.iterrows():
+            local_point = local_row.geometry
+            nearby_indices = osm_index.query(local_point.buffer(self.tolerance_deg))
+            
+            for osm_idx in nearby_indices:
+                osm_row = osm_gdf.iloc[osm_idx]
+                osm_point = osm_row.geometry
+                distance_meters = local_point.distance(osm_point) * 111000.0
+                
+                # Check if house numbers match (handle type differences)
+                local_house_num = str(local_row.get('addr:housenumber', ''))
+                osm_house_num = str(osm_row.get('addr:housenumber', ''))
+                
+                if (distance_meters <= self.tolerance_meters and 
+                    local_house_num == osm_house_num):
+                    matched_osm_indices.add(osm_idx)
+                    break  # Only match to first OSM address found
+        
+        # Find unmatched OSM addresses
+        for idx, osm_row in osm_gdf.iterrows():
+            if idx not in matched_osm_indices:
+                osm_props = dict(osm_row.drop('geometry'))
+                osm_props['status'] = 'removed'
+                removed_addresses.append({
+                    'geometry': osm_row.geometry,
+                    **osm_props
+                })
+        
+        return new_addresses, existing_addresses, removed_addresses
+    
+    def save_results(self, new_addresses: List[Dict], existing_addresses: List[Dict], 
+                    removed_addresses: List[Dict], output_dir: str):
+        """Save comparison results to separate GeoJSON files."""
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        
+        # Save new addresses (to add to OSM)
+        if new_addresses:
+            new_gdf = gpd.GeoDataFrame(new_addresses)
+            new_file = output_dir / f"addresses_to_add_{timestamp}.geojson"
+            new_gdf.to_file(new_file, driver='GeoJSON')
+            print(f"Saved {len(new_addresses)} new addresses to {new_file}")
+        
+        # Save removed addresses (missing from local data)
+        if removed_addresses:
+            removed_gdf = gpd.GeoDataFrame(removed_addresses)
+            removed_file = output_dir / f"addresses_potentially_removed_{timestamp}.geojson"
+            removed_gdf.to_file(removed_file, driver='GeoJSON')
+            print(f"Saved {len(removed_addresses)} potentially removed addresses to {removed_file}")
+        
+        # Save existing addresses for reference
+        if existing_addresses:
+            # Create simplified format for existing addresses
+            existing_simple = []
+            for addr in existing_addresses:
+                existing_simple.append({
+                    'geometry': addr['geometry'],
+                    'distance_meters': addr['distance_meters'],
+                    'status': 'existing'
+                })
+            
+            existing_gdf = gpd.GeoDataFrame(existing_simple)
+            existing_file = output_dir / f"addresses_existing_{timestamp}.geojson"
+            existing_gdf.to_file(existing_file, driver='GeoJSON')
+            print(f"Saved {len(existing_addresses)} existing addresses to {existing_file}")
+    
+    def print_summary(self, new_addresses: List[Dict], existing_addresses: List[Dict], 
+                     removed_addresses: List[Dict]):
+        """Print a summary of the comparison results."""
+        print("\n" + "="*60)
+        print("ADDRESS COMPARISON SUMMARY")
+        print("="*60)
+        
+        print(f"\nTOTAL ADDRESSES ANALYZED:")
+        print(f"  • New addresses (to add to OSM): {len(new_addresses)}")
+        print(f"  • Existing addresses (matched): {len(existing_addresses)}")
+        print(f"  • Potentially removed addresses: {len(removed_addresses)}")
+        
+        if existing_addresses:
+            distances = [addr['distance_meters'] for addr in existing_addresses]
+            avg_distance = sum(distances) / len(distances)
+            print(f"\nMATCHING STATISTICS:")
+            print(f"  • Average distance of matches: {avg_distance:.1f} meters")
+            print(f"  • Max distance of matches: {max(distances):.1f} meters")
+        
+        if new_addresses:
+            print(f"\nNEW ADDRESSES TO ADD:")
+            print(f"  These addresses exist in local data but not in OSM")
+            
+            # Group by street name
+            streets = {}
+            for addr in new_addresses[:10]:  # Show first 10
+                street = addr.get('addr:street', 'Unknown Street')
+                if street not in streets:
+                    streets[street] = 0
+                streets[street] += 1
+            
+            for street, count in sorted(streets.items()):
+                print(f"  • {street}: {count} address(es)")
+            
+            if len(new_addresses) > 10:
+                print(f"  • ... and {len(new_addresses) - 10} more")
+        
+        if removed_addresses:
+            print(f"\nPOTENTIALLY REMOVED ADDRESSES:")
+            print(f"  These addresses exist in OSM but not in local data")
+            print(f"  (May indicate addresses that were removed or demolished)")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Compare local government address data with OpenStreetMap addresses",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python compare-addresses.py "Lake" "Florida" --local-zip "original data/Lake/Addresspoints 2025-06.zip"
+  python compare-addresses.py "Sumter" "Florida" --local-zip "original data/Sumter/Address9_13_2024.zip" --tolerance 30
+  python compare-addresses.py "Orange" "Florida" --local-zip "addresses.zip" --output-dir "results/orange"
+        """
+    )
+    
+    parser.add_argument('county', help='County name (e.g., "Lake", "Sumter")')
+    parser.add_argument('state', help='State name (e.g., "Florida")')
+    parser.add_argument('--local-zip', required=True, help='Path to local address data ZIP file')
+    parser.add_argument('--tolerance', '-t', type=float, default=50.0,
+                       help='Distance tolerance in meters for matching addresses (default: 50)')
+    parser.add_argument('--output-dir', '-o', help='Output directory for results (default: processed data/[County])')
+    parser.add_argument('--cache-dir', default='osm_cache', 
+                       help='Directory to cache OSM downloads (default: osm_cache)')
+    parser.add_argument('--force-download', action='store_true',
+                       help='Force re-download of OSM data (ignore cache)')
+    parser.add_argument('--sample', '-s', type=int, 
+                       help='Process only a sample of N addresses for testing')
+    parser.add_argument('--max-osm', type=int, default=50000,
+                       help='Maximum number of OSM addresses to process (default: 50000)')
+    
+    args = parser.parse_args()
+    
+    # Validate input file
+    local_zip = Path(args.local_zip)
+    if not local_zip.exists():
+        print(f"Error: Local ZIP file {args.local_zip} does not exist")
+        return 1
+    
+    # Set output directory
+    if args.output_dir:
+        output_dir = Path(args.output_dir)
+    else:
+        output_dir = Path("processed data") / args.county
+    
+    try:
+        # Create comparator
+        comparator = AddressComparator(
+            tolerance_meters=args.tolerance,
+            cache_dir=args.cache_dir
+        )
+        
+        # Set sampling parameters
+        if args.sample:
+            comparator.sample_size = args.sample
+        if args.max_osm:
+            comparator.max_osm = args.max_osm
+        
+        # Download/load OSM data
+        if args.force_download:
+            # Remove existing cache for this county
+            for cache_file in Path(args.cache_dir).glob(f"osm_addresses_{args.county.lower()}_*.geojson"):
+                cache_file.unlink()
+        
+        osm_file = comparator.download_osm_addresses(args.county, args.state)
+        
+        # Convert local data
+        local_file = comparator.load_local_addresses(args.local_zip)
+        
+        # Perform comparison
+        new_addresses, existing_addresses, removed_addresses = comparator.compare_addresses(
+            local_file, osm_file
+        )
+        
+        # Save results
+        comparator.save_results(new_addresses, existing_addresses, removed_addresses, output_dir)
+        
+        # Print summary
+        comparator.print_summary(new_addresses, existing_addresses, removed_addresses)
+        
+        return 0
+        
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

diff-highways.py

@@ -54,22 +54,24 @@ def titlecase(s):
         s)
 
 class RoadComparator:
-    def __init__(self, tolerance_feet: float = 50.0, min_gap_length_feet: float = 100.0, 
-                 n_jobs: int = None, chunk_size: int = 200):
+    def __init__(self, tolerance_feet: float = 50.0, min_gap_length_feet: float = 100.0,
+                 n_jobs: int = None, chunk_size: int = 1000, exclude_unnamed: bool = False):
         """
         Initialize the road comparator.
-        
+
         Args:
             tolerance_feet: Distance tolerance for considering roads as overlapping (default: 50 feet)
             min_gap_length_feet: Minimum length of gap/extra to be considered significant (default: 100 feet)
-            n_jobs: Number of parallel processes to use (default: 2 for Windows)
-            chunk_size: Number of geometries to process per chunk (default: 200 for Windows)
+            n_jobs: Number of parallel processes to use (default: CPU count - 1)
+            chunk_size: Number of geometries to process per chunk (default: 1000)
+            exclude_unnamed: Exclude features without name/highway tags from coverage (default: False)
         """
         self.tolerance_feet = tolerance_feet
         self.min_gap_length_feet = min_gap_length_feet
         # Reduce worker count for Windows to prevent memory issues
         self.n_jobs = n_jobs or min(2, max(1, mp.cpu_count() // 2))
         self.chunk_size = chunk_size
+        self.exclude_unnamed = exclude_unnamed
         
         # Convert feet to degrees (approximate conversion for continental US)
         # 1 degree latitude ≈ 364,000 feet
@@ -78,8 +80,29 @@ class RoadComparator:
         self.min_gap_length_deg = min_gap_length_feet / 364000.0
         
         print(f"Using {self.n_jobs} parallel processes with chunk size {self.chunk_size}")
-    
-    def load_geojson(self, filepath: str) -> gpd.GeoDataFrame:
+        if self.exclude_unnamed:
+            print("Excluding unnamed features from coverage calculation")
+
+    def _has_name(self, row) -> bool:
+        """Check if a feature has a name tag (for OSM data filtering)."""
+        # Check for OSM-style tags (stored as JSON string)
+        if 'tags' in row.index:
+            tags = row.get('tags')
+            if isinstance(tags, dict):
+                return bool(tags.get('name'))
+            elif isinstance(tags, str):
+                # Tags stored as JSON string
+                try:
+                    tags_dict = json.loads(tags)
+                    return bool(tags_dict.get('name'))
+                except (json.JSONDecodeError, TypeError):
+                    return False
+            return False
+        # Check for direct name properties
+        name = row.get('name') or row.get('NAME') or row.get('FULLNAME')
+        return bool(name)
+
+    def load_geojson(self, filepath: str, filter_unnamed: bool = False) -> gpd.GeoDataFrame:
         """Load and validate GeoJSON file with optimizations."""
         try:
             # Use pyogr engine for faster loading of large files
@@ -100,7 +123,16 @@ class RoadComparator:
             if invalid_mask.any():
                 print(f"Fixing {invalid_mask.sum()} invalid geometries...")
                 gdf.loc[invalid_mask, 'geometry'] = gdf.loc[invalid_mask, 'geometry'].buffer(0)
-            
+
+            # Filter unnamed features if requested
+            if filter_unnamed:
+                original_count = len(gdf)
+                named_mask = gdf.apply(self._has_name, axis=1)
+                gdf = gdf[named_mask].copy()
+                gdf = gdf.reset_index(drop=True)
+                filtered_count = original_count - len(gdf)
+                print(f"Filtered out {filtered_count} unnamed features")
+
             print(f"Loaded {len(gdf)} road features from {filepath}")
             return gdf
             
@@ -111,8 +143,8 @@ class RoadComparator:
         """Create a buffered union using chunked processing for memory efficiency."""
         print("Creating optimized buffered union...")
         
-        # Process in chunks to manage memory
-        chunks = [gdf.iloc[i:i+self.chunk_size] for i in range(0, len(gdf), self.chunk_size)]
+        # Process in chunks to manage memory - extract geometries as lists
+        chunks = [gdf.iloc[i:i+self.chunk_size].geometry.tolist() for i in range(0, len(gdf), self.chunk_size)]
         chunk_unions = []
         
         # Use partial function for multiprocessing
@@ -147,17 +179,17 @@ class RoadComparator:
             raise Exception("No valid geometries to create union")
     
     @staticmethod
-    def _buffer_chunk(chunk_gdf: gpd.GeoDataFrame, tolerance: float) -> Any:
+    def _buffer_chunk(geometries: List, tolerance: float) -> Any:
         """Buffer geometries in a chunk and return their union."""
         try:
             # Buffer all geometries in the chunk
-            buffered = chunk_gdf.geometry.buffer(tolerance)
+            buffered = [geom.buffer(tolerance) for geom in geometries]
             
             # Create union of buffered geometries
             if len(buffered) == 1:
-                return buffered.iloc[0]
+                return buffered[0]
             else:
-                return unary_union(buffered.tolist())
+                return unary_union(buffered)
         except Exception as e:
             print(f"Error in chunk processing: {str(e)}")
             return None
@@ -177,9 +209,17 @@ class RoadComparator:
         """
         print("Finding removed segments...")
         
-        # Split into chunks for parallel processing
-        chunks = [source_gdf.iloc[i:i+self.chunk_size] 
-                 for i in range(0, len(source_gdf), self.chunk_size)]
+        # Split into chunks for parallel processing - convert to serializable format
+        chunks = []
+        for i in range(0, len(source_gdf), self.chunk_size):
+            chunk_gdf = source_gdf.iloc[i:i+self.chunk_size]
+            chunk_data = []
+            for idx, row in chunk_gdf.iterrows():
+                chunk_data.append({
+                    'geometry': row.geometry,
+                    'properties': dict(row.drop('geometry'))
+                })
+            chunks.append(chunk_data)
         
         all_removed = []
         
@@ -210,13 +250,14 @@ class RoadComparator:
         return all_removed
     
     @staticmethod
-    def _process_removed_chunk(chunk_gdf: gpd.GeoDataFrame, target_union: Any, 
+    def _process_removed_chunk(chunk_data: List[Dict], target_union: Any, 
                               min_length_deg: float) -> List[Dict[str, Any]]:
         """Process a chunk of geometries to find removed segments."""
         removed_segments = []
         
-        for idx, row in chunk_gdf.iterrows():
-            geom = row.geometry
+        for row_data in chunk_data:
+            geom = row_data['geometry']
+            properties = row_data['properties']
             
             # Handle MultiLineString by processing each component
             if isinstance(geom, MultiLineString):
@@ -245,12 +286,12 @@ class RoadComparator:
                     for uncovered_line in uncovered_lines:
                         if uncovered_line.length >= min_length_deg:
                             # Create properties dict with original metadata plus 'removed: true'
-                            properties = dict(row.drop('geometry'))
-                            properties['removed'] = True
+                            result_properties = properties.copy()
+                            result_properties['removed'] = True
                             
                             removed_segments.append({
                                 'geometry': uncovered_line,
-                                **properties
+                                **result_properties
                             })
                             
                 except Exception as e:
@@ -266,9 +307,17 @@ class RoadComparator:
         """
         print("Finding added roads...")
         
-        # Split into chunks for parallel processing
-        chunks = [source_gdf.iloc[i:i+self.chunk_size] 
-                 for i in range(0, len(source_gdf), self.chunk_size)]
+        # Split into chunks for parallel processing - convert to serializable format
+        chunks = []
+        for i in range(0, len(source_gdf), self.chunk_size):
+            chunk_gdf = source_gdf.iloc[i:i+self.chunk_size]
+            chunk_data = []
+            for idx, row in chunk_gdf.iterrows():
+                chunk_data.append({
+                    'geometry': row.geometry,
+                    'properties': dict(row.drop('geometry'))
+                })
+            chunks.append(chunk_data)
         
         all_added = []
         
@@ -299,13 +348,14 @@ class RoadComparator:
         return all_added
     
     @staticmethod
-    def _process_added_chunk(chunk_gdf: gpd.GeoDataFrame, target_union: Any, 
+    def _process_added_chunk(chunk_data: List[Dict], target_union: Any, 
                             min_length_deg: float) -> List[Dict[str, Any]]:
         """Process a chunk of geometries to find added roads."""
         added_roads = []
         
-        for idx, row in chunk_gdf.iterrows():
-            geom = row.geometry
+        for row_data in chunk_data:
+            geom = row_data['geometry']
+            original_properties = row_data['properties']
             
             try:
                 # Check what portion of the road is not covered
@@ -328,45 +378,59 @@ class RoadComparator:
                     # 1. The uncovered portion is above minimum threshold, AND
                     # 2. More than 10% of the road is uncovered
                     if uncovered_ratio > 0.1:
-                        #uncovered_length >= min_length_deg and 
+                        #uncovered_length >= min_length_deg and
                         # Include entire original road with all original metadata
-                        original_properties = dict(row.drop('geometry'))
 
-                        #
-                        # For Sumter County Roads
-                        #
                         properties = {
                             'surface': 'asphalt'
                         }
 
-                        for key, value in original_properties.items():
-                            if key == 'NAME':
-                                properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None
-                            elif key == 'SpeedLimit':
-                                properties['maxspeed'] = f"{value} mph" if value is not None else None
-                            elif key == 'RoadClass':
-                                if value is None:
-                                    properties['highway'] = 'residential'
-                                elif value.startswith('PRIMARY'):
-                                    properties['highway'] = 'trunk'
-                                elif value.startswith('MAJOR'):
-                                    properties['highway'] = 'primary'
-                                elif value.startswith('MINOR'):
-                                    properties['highway'] = 'secondary'
-                                elif value.startswith('SECONDARY'):
-                                    properties['highway'] = 'tertiary'
-                                elif value.startswith('TURN LANE'):
-                                    properties['highway'] = 'primary_link'
-                                # elif value.startswith('LOCAL'):
-                                else:
-                                    properties['highway'] = 'residential'
-                            elif lowercase(key) == 'oneway':
-                                properties['oneway'] = 'yes' if lowercase(value) == 'y' or lowercase(value) == 'ft' else None
-                            elif key == 'NumberOfLa':
-                                properties['lanes'] = value if value is not None else None
-                            # else:
-                            #     # Keep other properties as-is, or transform them as needed
-                            #     properties[key] = value
+                        # Detect county format based on available fields
+                        is_lake_county = 'FullStreet' in original_properties
+                        is_sumter_county = 'NAME' in original_properties and 'RoadClass' in original_properties
+
+                        if is_lake_county:
+                            # Lake County field mappings
+                            for key, value in original_properties.items():
+                                if key == 'FullStreet':
+                                    properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None
+                                elif key == 'SpeedLimit':
+                                    properties['maxspeed'] = f"{value} mph" if value is not None else None
+                                elif key == 'NumberOfLa':
+                                    try:
+                                        num_value = int(float(value)) if value is not None else 0
+                                        if num_value > 0:
+                                            properties['lanes'] = str(num_value)
+                                    except (ValueError, TypeError):
+                                        pass
+                                elif key == 'StreetClas':
+                                    highway_type = qgisfunctions.gethighwaytype(value, None, None)
+                                    properties['highway'] = highway_type if highway_type else 'residential'
+                        elif is_sumter_county:
+                            # Sumter County field mappings
+                            for key, value in original_properties.items():
+                                if key == 'NAME':
+                                    properties['name'] = titlecase(qgisfunctions.formatstreet(value,None,None)) if value is not None else None
+                                elif key == 'SpeedLimit':
+                                    properties['maxspeed'] = f"{value} mph" if value is not None else None
+                                elif key == 'RoadClass':
+                                    if value is None:
+                                        properties['highway'] = 'residential'
+                                    elif value.startswith('PRIMARY'):
+                                        properties['highway'] = 'trunk'
+                                    elif value.startswith('MAJOR'):
+                                        properties['highway'] = 'primary'
+                                    else:
+                                        properties['highway'] = 'residential'
+                        else:
+                            # Unknown format - try common field names
+                            name = original_properties.get('NAME') or original_properties.get('FullStreet') or original_properties.get('name')
+                            if name:
+                                properties['name'] = titlecase(qgisfunctions.formatstreet(name,None,None))
+                            speed = original_properties.get('SpeedLimit')
+                            if speed:
+                                properties['maxspeed'] = f"{speed} mph"
+                            properties['highway'] = 'residential'
 
                         added_roads.append({
                             'geometry': geom,
@@ -392,9 +456,10 @@ class RoadComparator:
         print(f"Minimum significant length: {self.min_gap_length_feet} feet")
         print(f"Parallel processing: {self.n_jobs} workers")
         print("-" * 50)
-        
+
         # Load both files
-        gdf1 = self.load_geojson(file1_path)
+        # Filter unnamed features from file1 (OSM data) if exclude_unnamed is set
+        gdf1 = self.load_geojson(file1_path, filter_unnamed=self.exclude_unnamed)
         gdf2 = self.load_geojson(file2_path)
         
         # Ensure both are in the same CRS
@@ -432,9 +497,20 @@ class RoadComparator:
         print("Saving results...")
         results_gdf = gpd.GeoDataFrame(all_results)
         
-        # Save to file with optimization
-        results_gdf.to_file(output_path, driver='GeoJSON', engine='pyogrio')
-        print(f"Results saved to: {output_path}")
+        # Save to file with optimization, with fallback for locked files
+        try:
+            results_gdf.to_file(output_path, driver='GeoJSON', engine='pyogrio')
+            print(f"Results saved to: {output_path}")
+        except (PermissionError, OSError) as e:
+            # File is locked, try with a timestamp suffix
+            from datetime import datetime
+            timestamp = datetime.now().strftime("%H%M%S")
+            base = Path(output_path)
+            fallback_path = str(base.parent / f"{base.stem}_{timestamp}{base.suffix}")
+            print(f"Warning: Could not save to {output_path}: {e}")
+            print(f"Saving to fallback: {fallback_path}")
+            results_gdf.to_file(fallback_path, driver='GeoJSON', engine='pyogrio')
+            print(f"Results saved to: {fallback_path}")
     
     def print_summary(self, removed: List[Dict], added: List[Dict], file1_name: str, file2_name: str):
         """Print a summary of the comparison results."""
@@ -448,7 +524,7 @@ class RoadComparator:
         print(f"Minimum significant length: {self.min_gap_length_feet} feet")
         
         if removed:
-            print(f"\n🔴 REMOVED ROADS ({len(removed)} segments):")
+            print(f"\nREMOVED ROADS ({len(removed)} segments):")
             print("These road segments exist in File 1 but are missing or incomplete in File 2:")
             
             # Calculate total length of removed segments
@@ -478,7 +554,7 @@ class RoadComparator:
                 print(f"  • {road}: {len(lengths)} segment(s), {road_total:,.1f} feet")
         
         if added:
-            print(f"\n🔵 ADDED ROADS ({len(added)} roads):")
+            print(f"\nADDED ROADS ({len(added)} roads):")
             print("These roads exist in File 2 but are missing or incomplete in File 1:")
             
             # Calculate total length of added roads
@@ -508,7 +584,7 @@ class RoadComparator:
                 print(f"  • {road}: {length:,.1f} feet")
         
         if not removed and not added:
-            print("\n✅ No significant differences found!")
+            print("\nNo significant differences found!")
             print("The road networks have good coverage overlap within the specified tolerance.")
 
 
@@ -536,7 +612,9 @@ Examples:
                        help='Number of parallel processes (default: CPU count - 1)')
     parser.add_argument('--chunk-size', '-c', type=int, default=1000,
                        help='Number of geometries to process per chunk (default: 1000)')
-    
+    parser.add_argument('--exclude-unnamed', '-e', action='store_true',
+                       help='Exclude features without name tags from coverage calculation (helps detect roads covered by unnamed geometry)')
+
     args = parser.parse_args()
     
     # Validate input files
@@ -554,7 +632,8 @@ Examples:
             tolerance_feet=args.tolerance,
             min_gap_length_feet=args.min_length,
             n_jobs=args.jobs,
-            chunk_size=args.chunk_size
+            chunk_size=args.chunk_size,
+            exclude_unnamed=args.exclude_unnamed
         )
         
         removed, added = comparator.compare_roads(args.file1, args.file2)

download-overpass.py

@@ -3,9 +3,9 @@
 Download OSM data from Overpass API for a given county and save as GeoJSON.
 
 Usage:
-    python download-overpass.py "Sumter County Florida" highways.geojson
-    python download-overpass.py "Lake County Florida" output/lake-addresses.geojson --type addresses
-    python download-overpass.py "Sumter County Florida" paths.geojson --type multimodal
+    python download-overpass.py --type highways "Sumter County" "Florida" output/roads.geojson
+    python download-overpass.py --type addresses "Lake County" "Florida" output/addresses.geojson
+    python download-overpass.py --type multimodal "Sumter County" "Florida" output/paths.geojson
 
 TODO:
 - Don't just download roads. Probably ignore relations also.
@@ -14,19 +14,57 @@ TODO:
 import argparse
 import json
 import sys
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from pathlib import Path
 
 
+def get_county_area_id(county_name, state_name):
+    """Get OSM area ID for a county using Nominatim."""
+    search_query = f"{county_name}, {state_name}, USA"
+    url = f"https://nominatim.openstreetmap.org/search?q={urllib.parse.quote(search_query)}&format=json&limit=1&featuretype=county"
+
+    # Nominatim requires User-Agent header
+    req = urllib.request.Request(url, headers={'User-Agent': 'TheVillagesImport/1.0'})
+
+    try:
+        with urllib.request.urlopen(req) as response:
+            results = json.loads(response.read().decode("utf-8"))
+
+        if results and results[0].get('osm_type') == 'relation':
+            relation_id = int(results[0]['osm_id'])
+            area_id = relation_id + 3600000000
+            print(f"Found {county_name}, {state_name}: relation {relation_id} -> area {area_id}")
+            return area_id
+
+        raise ValueError(f"Could not find relation for {county_name}, {state_name}")
+    except urllib.error.HTTPError as e:
+        print(f"Nominatim HTTP Error {e.code}: {e.reason}", file=sys.stderr)
+        sys.exit(1)
+
+
 def build_overpass_query(county_name, state_name, data_type="highways"):
     """Build Overpass API query for specified data type in a county."""
+    area_id = get_county_area_id(county_name, state_name)
+
     base_query = f"""[out:json][timeout:60];
-area["name"="{county_name}"]->.searchArea;"""
+area(id:{area_id})->.searchArea;"""
     
     if data_type == "highways":
-        selector = 'way["highway"](area.searchArea);'
+        selector = '('
+        selector += 'way["highway"="motorway"](area.searchArea);'
+        selector += 'way["highway"="trunk"](area.searchArea);'
+        selector += 'way["highway"="primary"](area.searchArea);'
+        selector += 'way["highway"="secondary"](area.searchArea);'
+        selector += 'way["highway"="tertiary"](area.searchArea);'
+        selector += 'way["highway"="unclassified"](area.searchArea);'
+        selector += 'way["highway"="residential"](area.searchArea);'
+        selector += 'way["highway"~"_link"](area.searchArea);'
+        selector += 'way["highway"="service"](area.searchArea);'
+        selector += 'way["highway"="track"](area.searchArea);'
+        selector += ');'
     elif data_type == "addresses":
         selector = 'nwr["addr:housenumber"](area.searchArea);'
     elif data_type == "multimodal":
@@ -128,6 +166,7 @@ def main():
     
     # Build and execute query
     query = build_overpass_query(args.county, args.state, args.type)
+    print(f"Query: {query}")
     overpass_data = query_overpass(query)
     
     # Convert to GeoJSON

processed data/Sumter/addresses-20240913.qmd

@@ -1,44 +0,0 @@
-<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
-<qgis version="3.34.1-Prizren">
-  <identifier></identifier>
-  <parentidentifier></parentidentifier>
-  <language></language>
-  <type>dataset</type>
-  <title></title>
-  <abstract></abstract>
-  <contact>
-    <name></name>
-    <organization></organization>
-    <position></position>
-    <voice></voice>
-    <fax></fax>
-    <email></email>
-    <role></role>
-  </contact>
-  <links/>
-  <dates/>
-  <fees></fees>
-  <encoding></encoding>
-  <crs>
-    <spatialrefsys nativeFormat="Wkt">
-      <wkt>PROJCRS["NAD83 / Florida West (ftUS)",BASEGEOGCRS["NAD83",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["Degree",0.0174532925199433]]],CONVERSION["unnamed",METHOD["Transverse Mercator",ID["EPSG",9807]],PARAMETER["Latitude of natural origin",24.3333333333333,ANGLEUNIT["Degree",0.0174532925199433],ID["EPSG",8801]],PARAMETER["Longitude of natural origin",-82,ANGLEUNIT["Degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Scale factor at natural origin",0.999941176470588,SCALEUNIT["unity",1],ID["EPSG",8805]],PARAMETER["False easting",656166.666666667,LENGTHUNIT["US survey foot",0.304800609601219],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["US survey foot",0.304800609601219],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["US survey foot",0.304800609601219,ID["EPSG",9003]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["US survey foot",0.304800609601219,ID["EPSG",9003]]]]</wkt>
-      <proj4>+proj=tmerc +lat_0=24.3333333333333 +lon_0=-82 +k=0.999941176470588 +x_0=200000 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=us-ft +no_defs +type=crs</proj4>
-      <srsid>0</srsid>
-      <srid>0</srid>
-      <authid></authid>
-      <description>NAD83 / Florida West (ftUS)</description>
-      <projectionacronym></projectionacronym>
-      <ellipsoidacronym>PARAMETER:6378137:6356752.31414035614579916</ellipsoidacronym>
-      <geographicflag>false</geographicflag>
-    </spatialrefsys>
-  </crs>
-  <extent>
-    <spatial minx="179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" maxx="-179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" crs="" miny="179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" maxy="-179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" minz="0" maxz="0" dimensions="2"/>
-    <temporal>
-      <period>
-        <start></start>
-        <end></end>
-      </period>
-    </temporal>
-  </extent>
-</qgis>

processed data/Sumter/addresses-20250124.qmd

@@ -1,44 +0,0 @@
-<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
-<qgis version="3.34.1-Prizren">
-  <identifier></identifier>
-  <parentidentifier></parentidentifier>
-  <language></language>
-  <type>dataset</type>
-  <title></title>
-  <abstract></abstract>
-  <contact>
-    <name></name>
-    <organization></organization>
-    <position></position>
-    <voice></voice>
-    <fax></fax>
-    <email></email>
-    <role></role>
-  </contact>
-  <links/>
-  <dates/>
-  <fees></fees>
-  <encoding></encoding>
-  <crs>
-    <spatialrefsys nativeFormat="Wkt">
-      <wkt>PROJCRS["NAD83 / Florida West (ftUS)",BASEGEOGCRS["NAD83",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["Degree",0.0174532925199433]]],CONVERSION["unnamed",METHOD["Transverse Mercator",ID["EPSG",9807]],PARAMETER["Latitude of natural origin",24.3333333333333,ANGLEUNIT["Degree",0.0174532925199433],ID["EPSG",8801]],PARAMETER["Longitude of natural origin",-82,ANGLEUNIT["Degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Scale factor at natural origin",0.999941176470588,SCALEUNIT["unity",1],ID["EPSG",8805]],PARAMETER["False easting",656166.666666667,LENGTHUNIT["US survey foot",0.304800609601219],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["US survey foot",0.304800609601219],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["US survey foot",0.304800609601219,ID["EPSG",9003]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["US survey foot",0.304800609601219,ID["EPSG",9003]]]]</wkt>
-      <proj4>+proj=tmerc +lat_0=24.3333333333333 +lon_0=-82 +k=0.999941176470588 +x_0=200000 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=us-ft +no_defs +type=crs</proj4>
-      <srsid>0</srsid>
-      <srid>0</srid>
-      <authid></authid>
-      <description>NAD83 / Florida West (ftUS)</description>
-      <projectionacronym></projectionacronym>
-      <ellipsoidacronym>PARAMETER:6378137:6356752.31414035614579916</ellipsoidacronym>
-      <geographicflag>false</geographicflag>
-    </spatialrefsys>
-  </crs>
-  <extent>
-    <spatial crs="" dimensions="2" minx="179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" maxx="-179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" miny="179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" maxy="-179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" minz="0" maxz="0"/>
-    <temporal>
-      <period>
-        <start></start>
-        <end></end>
-      </period>
-    </temporal>
-  </extent>
-</qgis>

processed data/Sumter/roads-20240913.qmd

@@ -1,44 +0,0 @@
-<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
-<qgis version="3.34.1-Prizren">
-  <identifier></identifier>
-  <parentidentifier></parentidentifier>
-  <language></language>
-  <type>dataset</type>
-  <title></title>
-  <abstract></abstract>
-  <contact>
-    <name></name>
-    <organization></organization>
-    <position></position>
-    <voice></voice>
-    <fax></fax>
-    <email></email>
-    <role></role>
-  </contact>
-  <links/>
-  <dates/>
-  <fees></fees>
-  <encoding></encoding>
-  <crs>
-    <spatialrefsys nativeFormat="Wkt">
-      <wkt>PROJCRS["NAD83 / Florida West (ftUS)",BASEGEOGCRS["NAD83",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["Degree",0.0174532925199433]]],CONVERSION["unnamed",METHOD["Transverse Mercator",ID["EPSG",9807]],PARAMETER["Latitude of natural origin",24.3333333333333,ANGLEUNIT["Degree",0.0174532925199433],ID["EPSG",8801]],PARAMETER["Longitude of natural origin",-82,ANGLEUNIT["Degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Scale factor at natural origin",0.999941176470588,SCALEUNIT["unity",1],ID["EPSG",8805]],PARAMETER["False easting",656166.666666667,LENGTHUNIT["US survey foot",0.304800609601219],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["US survey foot",0.304800609601219],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["US survey foot",0.304800609601219,ID["EPSG",9003]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["US survey foot",0.304800609601219,ID["EPSG",9003]]]]</wkt>
-      <proj4>+proj=tmerc +lat_0=24.3333333333333 +lon_0=-82 +k=0.999941176470588 +x_0=200000 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=us-ft +no_defs +type=crs</proj4>
-      <srsid>0</srsid>
-      <srid>0</srid>
-      <authid></authid>
-      <description>NAD83 / Florida West (ftUS)</description>
-      <projectionacronym></projectionacronym>
-      <ellipsoidacronym>PARAMETER:6378137:6356752.31414035614579916</ellipsoidacronym>
-      <geographicflag>false</geographicflag>
-    </spatialrefsys>
-  </crs>
-  <extent>
-    <spatial minx="179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" maxx="-179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" crs="" miny="179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" maxy="-179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" minz="0" maxz="0" dimensions="2"/>
-    <temporal>
-      <period>
-        <start></start>
-        <end></end>
-      </period>
-    </temporal>
-  </extent>
-</qgis>

run-lake-addresses.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""
+Simple wrapper script for comparing Lake County addresses
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+def main():
+    # Change to script directory
+    script_dir = Path(__file__).parent
+    
+    # Define the command
+    cmd = [
+        sys.executable, 
+        "compare-addresses.py",
+        "Lake", 
+        "Florida",
+        "--local-zip", "original data/Lake/Addresspoints 2025-06.zip",
+        "--tolerance", "50",
+        "--output-dir", "processed data/Lake"
+    ]
+    
+    print("Running Lake County address comparison...")
+    print("Command:", " ".join(cmd))
+    print()
+    
+    # Run the command
+    result = subprocess.run(cmd, cwd=script_dir)
+    
+    if result.returncode == 0:
+        print("\nAddress comparison completed successfully!")
+        print("Results saved in: processed data/Lake/")
+    else:
+        print(f"\nError: Script failed with return code {result.returncode}")
+    
+    return result.returncode
+
+if __name__ == "__main__":
+    sys.exit(main())