#!/usr/bin/env python3 """ Compare OSM-formatted county address GeoJSON against OpenStreetMap addresses. Expects both inputs to already be in OSM field format (addr:housenumber, addr:street, etc.). Use convert-addresses.py to prepare the county file and download-overpass.py for the OSM file. Usage: python compare-addresses.py \ --local-file /data/sumter/county-addresses.geojson \ --osm-file /data/sumter/osm-addresses.geojson \ --output-dir /data/sumter """ import argparse import re import subprocess import sys from pathlib import Path from typing import Dict, List, Tuple import geopandas as gpd import numpy as np import pandas as pd from shapely.strtree import STRtree import warnings warnings.filterwarnings('ignore') CONFLICT_RADIUS_METERS = 5.0 RENAME_RADIUS_METERS = 100.0 class AddressComparator: def __init__(self, tolerance_meters: float = 500.0): self.tolerance_meters = tolerance_meters # 1 degree latitude ≈ 111,000 metres self.tolerance_deg = tolerance_meters / 111000.0 @staticmethod def _normalize_multi(value) -> str: """Normalise a semicolon-delimited field by sorting its parts. Treats '101;201' and '201;101' as equal so ordering differences don't create false positives. """ if value is None or (isinstance(value, float) and pd.isna(value)): return '' s = str(value).strip() if not s or s == 'nan': return '' parts = sorted(p.strip().lower() for p in s.split(';') if p.strip()) return ';'.join(parts) def _normalize_street_name(self, street: str) -> str: """Normalise street names for fuzzy matching across county/OSM formatting differences.""" if not street or street == 'nan': return '' s = street.strip().lower() s = re.sub(r'\bstate road\b', 'sr', s) s = re.sub(r'\bstate route\b', 'sr', s) s = re.sub(r'\bcounty road\b', 'cr', s) s = re.sub(r'\bc\b', 'cr', s) # "C 44a" -> "cr 44a" # normalise CR sub-segment separators: "cr 109d 1" -> "cr 109d-1" s = re.sub(r'\b(cr\s+\d+[a-z])\s+(\d+)', r'\1-\2', s) s = re.sub(r'\s+', ' ', s).strip() return s @staticmethod def _flag_close_conflicts(addresses: List[Dict], radius_meters: float = CONFLICT_RADIUS_METERS) -> None: """Mark each address dict's 'conflict' key True when another point in the same list falls within radius_meters. Doesn't dedupe or exclude anything - just surfaces tight clusters (e.g. ambiguous source data reusing the same unit label across distinct points) for manual review in the map UI.""" if not addresses: return geoms = [a['geometry'] for a in addresses] tree = STRtree(geoms) radius_deg = radius_meters / 111000.0 for i, addr in enumerate(addresses): conflict = False for j in tree.query(geoms[i].buffer(radius_deg)): if j == i: continue if geoms[i].distance(geoms[j]) * 111000.0 <= radius_meters: conflict = True break addr['conflict'] = conflict def _flag_possible_renames(self, new_addresses: List[Dict], removed_addresses: List[Dict], radius_meters: float = RENAME_RADIUS_METERS) -> int: """Pair each new county address with an unmatched OSM address nearby that has the same house number (and a compatible unit) but a different street name, e.g. county "828 Donelin Circle" vs OSM "828 Donelin CR". Both sides get 'possible_rename' set to the other side's street so the map can show them as a likely rename/misnaming rather than an unrelated add + removal, and a shared 'rename_pair' id so the map can accept both together. Returns the number of pairs found.""" if not new_addresses or not removed_addresses: return 0 # Index the unmatched OSM side by house number; a pair needs the same one. geoms = [a['geometry'] for a in removed_addresses] by_num: Dict[str, List[int]] = {} for j, osm in enumerate(removed_addresses): by_num.setdefault(self._normalize_multi(osm.get('addr:housenumber')), []).append(j) street_cache: Dict[int, str] = {} paired = set() count = 0 for addr in new_addresses: num = self._normalize_multi(addr.get('addr:housenumber')) street = self._normalize_street_name(str(addr.get('addr:street', ''))) unit = self._normalize_multi(addr.get('addr:unit')) if not num or not street: continue best, best_dist = None, float('inf') for j in by_num.get(num, []): if j in paired: continue osm = removed_addresses[j] if j not in street_cache: street_cache[j] = self._normalize_street_name(str(osm.get('addr:street', ''))) osm_street = street_cache[j] if not osm_street or osm_street == street: continue osm_unit = self._normalize_multi(osm.get('addr:unit')) if unit and osm_unit and unit != osm_unit: continue d = addr['geometry'].distance(geoms[j]) * 111000.0 if d <= radius_meters and d < best_dist: best, best_dist = j, d if best is not None: paired.add(best) osm = removed_addresses[best] addr['possible_rename'] = str(osm.get('addr:street')) osm['possible_rename'] = str(addr.get('addr:street')) count += 1 addr['rename_pair'] = osm['rename_pair'] = count return count def compare_addresses(self, local_file: str, osm_file: str) -> Tuple[List[Dict], List[Dict], List[Dict]]: """Compare local and OSM address data. Returns: (new_addresses, existing_addresses, removed_addresses) """ print(f"Comparing addresses: {local_file} vs {osm_file}") local_gdf = gpd.read_file(local_file) osm_gdf = gpd.read_file(osm_file) print(f"Loaded local addresses: {len(local_gdf)}") print(f"Loaded OSM addresses: {len(osm_gdf)}") if hasattr(self, 'sample_size') and self.sample_size: if len(local_gdf) > self.sample_size: local_gdf = local_gdf.sample(n=self.sample_size, random_state=42).reset_index(drop=True) print(f"Sampled local addresses to: {len(local_gdf)}") if hasattr(self, 'max_osm') and self.max_osm: if len(osm_gdf) > self.max_osm: osm_gdf = osm_gdf.sample(n=self.max_osm, random_state=42).reset_index(drop=True) print(f"Sampled OSM addresses to: {len(osm_gdf)}") if local_gdf.crs != osm_gdf.crs: osm_gdf = osm_gdf.to_crs(local_gdf.crs) # Normalise every address once and index OSM by (number, street): # a county address can only ever match an OSM address with the same # key, so there's no need to scan everything within the tolerance # (thousands of points in dense areas like The Villages). def keys(gdf): nums = [self._normalize_multi(v) for v in gdf.get('addr:housenumber', pd.Series([''] * len(gdf)))] streets = [self._normalize_street_name(str(v)) for v in gdf.get('addr:street', pd.Series([''] * len(gdf)))] units = [self._normalize_multi(v) for v in gdf.get('addr:unit', pd.Series([None] * len(gdf)))] return nums, streets, units osm_nums, osm_streets, osm_units = keys(osm_gdf) local_nums, local_streets, local_units = keys(local_gdf) osm_geoms = osm_gdf.geometry.tolist() local_geoms = local_gdf.geometry.tolist() osm_props = osm_gdf.drop(columns='geometry').to_dict('records') local_props = local_gdf.drop(columns='geometry').to_dict('records') osm_by_key: Dict[Tuple[str, str], List[int]] = {} for i, (num, street) in enumerate(zip(osm_nums, osm_streets)): if street: osm_by_key.setdefault((num, street), []).append(i) existing_addresses = [] new_addresses = [] matched_osm = set() for i, local_point in enumerate(local_geoms): street = local_streets[i] candidates = osm_by_key.get((local_nums[i], street), []) if street else [] local_unit = local_units[i] # Existing: nearest OSM address with the same unit (or both # without one) within the tolerance. best_match, min_distance = None, float('inf') # Matched (not potentially removed): the same, except an OSM # address with no unit also covers a county address with one. nearest_cover, cover_distance = None, float('inf') for j in candidates: distance = local_point.distance(osm_geoms[j]) if distance * 111000.0 > self.tolerance_meters: continue osm_unit = osm_units[j] if (local_unit == osm_unit) and distance < min_distance: best_match, min_distance = j, distance if (not local_unit or not osm_unit or local_unit == osm_unit) and distance < cover_distance: nearest_cover, cover_distance = j, distance # Prefer marking the exact match, so e.g. a unit's own OSM node # counts as matched rather than the unit-less building around it cover = best_match if best_match is not None else nearest_cover if cover is not None: matched_osm.add(cover) if best_match is not None: existing_addresses.append({ 'geometry': local_point, 'osm_geometry': osm_geoms[best_match], 'local_data': local_props[i], 'osm_data': osm_props[best_match], 'distance_meters': min_distance * 111000.0, }) else: props = dict(local_props[i]) props['status'] = 'new' new_addresses.append({'geometry': local_point, **props}) removed_addresses = [] for j, osm_point in enumerate(osm_geoms): if j not in matched_osm: props = dict(osm_props[j]) # geopandas reads JSON int arrays (osm_nodes) as numpy arrays, # which would be written back out as a string for k, v in props.items(): if isinstance(v, np.ndarray): props[k] = [int(x) for x in v.tolist()] props['status'] = 'removed' removed_addresses.append({'geometry': osm_point, **props}) self._flag_close_conflicts(new_addresses) renames = self._flag_possible_renames(new_addresses, removed_addresses) print(f"Possible street renames (same number nearby, different street): {renames}") return new_addresses, existing_addresses, removed_addresses def save_results(self, new_addresses, existing_addresses, removed_addresses, output_dir): output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) if new_addresses: gpd.GeoDataFrame(new_addresses).to_file( output_dir / "addresses-to-add.geojson", driver='GeoJSON') print(f"Saved {len(new_addresses)} new addresses to {output_dir}/addresses-to-add.geojson") if removed_addresses: gpd.GeoDataFrame(removed_addresses).to_file( output_dir / "addresses-potentially-removed.geojson", driver='GeoJSON') print(f"Saved {len(removed_addresses)} potentially removed to {output_dir}/addresses-potentially-removed.geojson") if existing_addresses: # Keeps both county city fields (postal community and legal # city/jurisdiction - they can differ) plus OSM's addr:city, and # the OSM point's coordinates, so the "Generate City Boundaries # Report" step can build its report from this file alone without # repeating the spatial match. existing_simple = [ { 'geometry': a.get('osm_geometry') or a['geometry'], 'distance_meters': a['distance_meters'], 'addr:housenumber': a['local_data'].get('addr:housenumber'), 'addr:street': a['local_data'].get('addr:street'), 'addr:unit': a['local_data'].get('addr:unit'), 'county_postal_community': a['local_data'].get('county_postal_community'), 'county_city': a['local_data'].get('county_city'), 'osm_city': a['osm_data'].get('addr:city'), } for a in existing_addresses ] gpd.GeoDataFrame(existing_simple).to_file( output_dir / "addresses-existing.geojson", driver='GeoJSON') print(f"Saved {len(existing_addresses)} existing addresses to {output_dir}/addresses-existing.geojson") def print_summary(self, new_addresses, existing_addresses, removed_addresses): print("\n" + "=" * 60) print("ADDRESS COMPARISON SUMMARY") print("=" * 60) print(f"\n New (to add to OSM): {len(new_addresses)}") print(f" Existing (matched): {len(existing_addresses)}") print(f" Potentially removed: {len(removed_addresses)}") if existing_addresses: distances = [a['distance_meters'] for a in existing_addresses] print(f"\n Avg match distance: {sum(distances)/len(distances):.1f} m") print(f" Max match distance: {max(distances):.1f} m") if new_addresses: print(f"\n Sample new addresses by street:") streets: Dict[str, int] = {} for addr in new_addresses[:10]: s = addr.get('addr:street', 'Unknown') streets[s] = streets.get(s, 0) + 1 for s, n in sorted(streets.items()): print(f" {s}: {n}") if len(new_addresses) > 10: print(f" ... and {len(new_addresses) - 10} more") def main(): parser = argparse.ArgumentParser( description="Compare OSM-formatted county addresses against OpenStreetMap", epilog=""" Run convert-addresses.py first to produce county-addresses.geojson, and download-overpass.py to produce osm-addresses.geojson. """ ) parser.add_argument('--local-file', required=True, help='Pre-converted county addresses GeoJSON (from convert-addresses.py)') parser.add_argument('--osm-file', required=True, help='Downloaded OSM addresses GeoJSON (from download-overpass.py)') parser.add_argument('--output-dir', '-o', required=True, help='Output directory for result GeoJSON files') parser.add_argument('--tolerance', '-t', type=float, default=500.0, help='Spatial tolerance in metres for matching (default: 500)') parser.add_argument('--sample', '-s', type=int, help='Process only N local addresses (for testing)') parser.add_argument('--max-osm', type=int, help='Cap OSM addresses loaded (for testing)') args = parser.parse_args() local_file = Path(args.local_file) osm_file = Path(args.osm_file) output_dir = Path(args.output_dir) if not local_file.exists(): print(f"Error: local file not found: {local_file}", file=sys.stderr) print("Run convert-addresses.py first.", file=sys.stderr) return 1 if not osm_file.exists(): print(f"Error: OSM file not found: {osm_file}", file=sys.stderr) print("Run download-overpass.py first.", file=sys.stderr) return 1 comparator = AddressComparator(tolerance_meters=args.tolerance) if args.sample: comparator.sample_size = args.sample if args.max_osm: comparator.max_osm = args.max_osm new, existing, removed = comparator.compare_addresses(str(local_file), str(osm_file)) comparator.save_results(new, existing, removed, output_dir) comparator.print_summary(new, existing, removed) return 0 if __name__ == '__main__': sys.exit(main())