will/osm-import-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add LIFECYCLE filtering, street exceptions UI, address matching improvements, map viewer fixes

Browse Source
This commit is contained in:
zyphlar
2026-04-09 09:36:14 -07:00
parent 449a182fa4
commit e8267b1eee
10 changed files with 544 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compare-addresses.py
+66 -19
View File
@@ -57,6 +57,20 @@ class AddressComparator:
# 1 degree latitude ≈ 111,000 meters
self.tolerance_deg = tolerance_meters / 111000.0
# Load street name exceptions from YAML
exceptions_path = Path('/data/exceptions.yml')
if exceptions_path.exists():
import yaml
with open(exceptions_path) as f:
data = yaml.safe_load(f) or {}
self._exceptions = {
item['from']: item['to']
for item in data.get('corrections', [])
if 'from' in item and 'to' in item
}
else:
self._exceptions = {}
def _get_county_area_id(self, county: str, state: str) -> int:
"""Get OSM area ID for a county using Nominatim."""
search_query = f"{county} County, {state}, USA"
@@ -230,12 +244,24 @@ out geom;"""
# Load and process shapefile
gdf = gpd.read_file(shp_file)
# Filter to active records only (Sumter uses LIFECYCLE field)
LIFECYCLE_FIELD = 'LIFECYCLE'
ACTIVE_VALUE = 'Current'
if LIFECYCLE_FIELD in gdf.columns:
before = len(gdf)
gdf = gdf[gdf[LIFECYCLE_FIELD] == ACTIVE_VALUE].copy().reset_index(drop=True)
filtered = before - len(gdf)
if filtered:
print(f"Filtered out {filtered} non-active addresses (LIFECYCLE != '{ACTIVE_VALUE}')")
else:
print(f"All {len(gdf)} addresses are active (LIFECYCLE == '{ACTIVE_VALUE}')")
# Convert CRS to WGS84 if needed
if gdf.crs and gdf.crs != 'EPSG:4326':
print(f"Converting from {gdf.crs} to EPSG:4326")
gdf = gdf.to_crs('EPSG:4326')
# Process address fields using existing logic from sumter-address-convert.py
gdf = self._process_address_fields(gdf)
@@ -343,6 +369,13 @@ out geom;"""
address_mapping['addr:street'] = street_names
# Apply street name exceptions (e.g. county data quirks fixed for comparison/import)
if 'addr:street' in address_mapping and self._exceptions:
address_mapping['addr:street'] = [
self._exceptions.get(name, name) if name is not None else None
for name in address_mapping['addr:street']
]
# City - try multiple field names
city_fields = ['POST_COMM', 'PostalCity', 'CITY', 'Jurisdicti']
for field in city_fields:
@@ -375,6 +408,21 @@ out geom;"""
return processed_gdf
@staticmethod
def _normalize_multi(value) -> str:
"""Normalize a semicolon-delimited field by sorting its parts.
Treats '101;201;301' and '301;101;201' as equal — only real
content differences (not ordering) count as added/removed.
"""
if value is None or (isinstance(value, float) and pd.isna(value)):
return ''
s = str(value).strip()
if not s or s == 'nan':
return ''
parts = sorted(p.strip().lower() for p in s.split(';') if p.strip())
return ';'.join(parts)
def _normalize_street_name(self, street: str) -> str:
"""
Normalize street names for better matching.
@@ -464,8 +512,9 @@ out geom;"""
distance = local_point.distance(osm_point)
# Verify house number, street, and unit (if present) match
local_house_num = str(local_row.get('addr:housenumber', ''))
osm_house_num = str(osm_row.get('addr:housenumber', ''))
# _normalize_multi sorts semicolon-delimited values so order doesn't matter
local_house_num = self._normalize_multi(local_row.get('addr:housenumber', ''))
osm_house_num = self._normalize_multi(osm_row.get('addr:housenumber', ''))
# Check street name match (required) - use normalization for better matching
local_street = self._normalize_street_name(str(local_row.get('addr:street', '')))
@@ -473,16 +522,15 @@ out geom;"""
street_match = (local_street == osm_street and local_street != '')
# Check unit match - if either has a unit, both must match
local_unit = local_row.get('addr:unit')
osm_unit = osm_row.get('addr:unit')
local_unit_norm = self._normalize_multi(local_row.get('addr:unit'))
osm_unit_norm = self._normalize_multi(osm_row.get('addr:unit'))
# Determine if each side has a unit
local_has_unit = local_unit is not None and pd.notna(local_unit) and str(local_unit).strip() != ''
osm_has_unit = osm_unit is not None and pd.notna(osm_unit) and str(osm_unit).strip() != ''
local_has_unit = bool(local_unit_norm)
osm_has_unit = bool(osm_unit_norm)
if local_has_unit and osm_has_unit:
# Both have units - they must match
unit_match = (str(local_unit).strip().lower() == str(osm_unit).strip().lower())
# Both have units - they must match (order-insensitive)
unit_match = (local_unit_norm == osm_unit_norm)
elif local_has_unit or osm_has_unit:
# One has unit, other doesn't - no match
unit_match = False
@@ -538,21 +586,20 @@ out geom;"""
distance_meters = local_point.distance(osm_point) * 111000.0
# Verify house number, street, and unit (if present) match
local_house_num = str(local_row.get('addr:housenumber', ''))
osm_house_num = str(osm_row.get('addr:housenumber', ''))
local_house_num = self._normalize_multi(local_row.get('addr:housenumber', ''))
osm_house_num = self._normalize_multi(osm_row.get('addr:housenumber', ''))
# Check street name match (required) - use normalization for better matching
local_street = self._normalize_street_name(str(local_row.get('addr:street', '')))
osm_street = self._normalize_street_name(str(osm_row.get('addr:street', '')))
street_match = (local_street == osm_street and local_street != '')
# Check unit match (only if both have units specified)
local_unit = local_row.get('addr:unit')
osm_unit = osm_row.get('addr:unit')
# Check unit match (order-insensitive via _normalize_multi)
local_unit_norm = self._normalize_multi(local_row.get('addr:unit'))
osm_unit_norm = self._normalize_multi(osm_row.get('addr:unit'))
unit_match = True
if local_unit is not None and pd.notna(local_unit) and osm_unit is not None and pd.notna(osm_unit):
# Both have units - they must match
unit_match = (str(local_unit).strip().lower() == str(osm_unit).strip().lower())
if local_unit_norm and osm_unit_norm:
unit_match = (local_unit_norm == osm_unit_norm)
if (distance_meters <= self.tolerance_meters and
local_house_num == osm_house_num and