Speed up address and road diffs from hours to seconds, gzip GeoJSON downloads, and render map points on canvas with viewport-only OSM/county layers
This commit is contained in:
zyphlar
+65
-76
@@ -99,9 +99,12 @@ class AddressComparator:
|
||||
Returns the number of pairs found."""
|
||||
if not new_addresses or not removed_addresses:
|
||||
return 0
|
||||
# Index the unmatched OSM side by house number; a pair needs the same one.
|
||||
geoms = [a['geometry'] for a in removed_addresses]
|
||||
tree = STRtree(geoms)
|
||||
radius_deg = radius_meters / 111000.0
|
||||
by_num: Dict[str, List[int]] = {}
|
||||
for j, osm in enumerate(removed_addresses):
|
||||
by_num.setdefault(self._normalize_multi(osm.get('addr:housenumber')), []).append(j)
|
||||
street_cache: Dict[int, str] = {}
|
||||
paired = set()
|
||||
count = 0
|
||||
for addr in new_addresses:
|
||||
@@ -111,13 +114,13 @@ class AddressComparator:
|
||||
if not num or not street:
|
||||
continue
|
||||
best, best_dist = None, float('inf')
|
||||
for j in tree.query(addr['geometry'].buffer(radius_deg)):
|
||||
for j in by_num.get(num, []):
|
||||
if j in paired:
|
||||
continue
|
||||
osm = removed_addresses[j]
|
||||
if self._normalize_multi(osm.get('addr:housenumber')) != num:
|
||||
continue
|
||||
osm_street = self._normalize_street_name(str(osm.get('addr:street', '')))
|
||||
if j not in street_cache:
|
||||
street_cache[j] = self._normalize_street_name(str(osm.get('addr:street', '')))
|
||||
osm_street = street_cache[j]
|
||||
if not osm_street or osm_street == street:
|
||||
continue
|
||||
osm_unit = self._normalize_multi(osm.get('addr:unit'))
|
||||
@@ -162,97 +165,83 @@ class AddressComparator:
|
||||
if local_gdf.crs != osm_gdf.crs:
|
||||
osm_gdf = osm_gdf.to_crs(local_gdf.crs)
|
||||
|
||||
osm_index = STRtree(osm_gdf.geometry.tolist())
|
||||
# Normalise every address once and index OSM by (number, street):
|
||||
# a county address can only ever match an OSM address with the same
|
||||
# key, so there's no need to scan everything within the tolerance
|
||||
# (thousands of points in dense areas like The Villages).
|
||||
def keys(gdf):
|
||||
nums = [self._normalize_multi(v) for v in gdf.get('addr:housenumber', pd.Series([''] * len(gdf)))]
|
||||
streets = [self._normalize_street_name(str(v)) for v in gdf.get('addr:street', pd.Series([''] * len(gdf)))]
|
||||
units = [self._normalize_multi(v) for v in gdf.get('addr:unit', pd.Series([None] * len(gdf)))]
|
||||
return nums, streets, units
|
||||
|
||||
osm_nums, osm_streets, osm_units = keys(osm_gdf)
|
||||
local_nums, local_streets, local_units = keys(local_gdf)
|
||||
osm_geoms = osm_gdf.geometry.tolist()
|
||||
local_geoms = local_gdf.geometry.tolist()
|
||||
osm_props = osm_gdf.drop(columns='geometry').to_dict('records')
|
||||
local_props = local_gdf.drop(columns='geometry').to_dict('records')
|
||||
|
||||
osm_by_key: Dict[Tuple[str, str], List[int]] = {}
|
||||
for i, (num, street) in enumerate(zip(osm_nums, osm_streets)):
|
||||
if street:
|
||||
osm_by_key.setdefault((num, street), []).append(i)
|
||||
|
||||
existing_addresses = []
|
||||
new_addresses = []
|
||||
matched_osm = set()
|
||||
|
||||
for _, local_row in local_gdf.iterrows():
|
||||
local_point = local_row.geometry
|
||||
nearby = osm_index.query(local_point.buffer(self.tolerance_deg))
|
||||
for i, local_point in enumerate(local_geoms):
|
||||
street = local_streets[i]
|
||||
candidates = osm_by_key.get((local_nums[i], street), []) if street else []
|
||||
local_unit = local_units[i]
|
||||
|
||||
best_match = None
|
||||
min_distance = float('inf')
|
||||
# Existing: nearest OSM address with the same unit (or both
|
||||
# without one) within the tolerance.
|
||||
best_match, min_distance = None, float('inf')
|
||||
# Matched (not potentially removed): the same, except an OSM
|
||||
# address with no unit also covers a county address with one.
|
||||
nearest_cover, cover_distance = None, float('inf')
|
||||
for j in candidates:
|
||||
distance = local_point.distance(osm_geoms[j])
|
||||
if distance * 111000.0 > self.tolerance_meters:
|
||||
continue
|
||||
osm_unit = osm_units[j]
|
||||
if (local_unit == osm_unit) and distance < min_distance:
|
||||
best_match, min_distance = j, distance
|
||||
if (not local_unit or not osm_unit or local_unit == osm_unit) and distance < cover_distance:
|
||||
nearest_cover, cover_distance = j, distance
|
||||
|
||||
for osm_idx in nearby:
|
||||
osm_row = osm_gdf.iloc[osm_idx]
|
||||
distance = local_point.distance(osm_row.geometry)
|
||||
# Prefer marking the exact match, so e.g. a unit's own OSM node
|
||||
# counts as matched rather than the unit-less building around it
|
||||
cover = best_match if best_match is not None else nearest_cover
|
||||
if cover is not None:
|
||||
matched_osm.add(cover)
|
||||
|
||||
local_num = self._normalize_multi(local_row.get('addr:housenumber', ''))
|
||||
osm_num = self._normalize_multi(osm_row.get('addr:housenumber', ''))
|
||||
|
||||
local_street = self._normalize_street_name(str(local_row.get('addr:street', '')))
|
||||
osm_street = self._normalize_street_name(str(osm_row.get('addr:street', '')))
|
||||
street_match = local_street == osm_street and local_street != ''
|
||||
|
||||
local_unit = self._normalize_multi(local_row.get('addr:unit'))
|
||||
osm_unit = self._normalize_multi(osm_row.get('addr:unit'))
|
||||
|
||||
if local_unit and osm_unit:
|
||||
unit_match = local_unit == osm_unit
|
||||
elif local_unit or osm_unit:
|
||||
unit_match = False
|
||||
else:
|
||||
unit_match = True
|
||||
|
||||
if local_num == osm_num and street_match and unit_match and distance < min_distance:
|
||||
min_distance = distance
|
||||
best_match = osm_idx
|
||||
|
||||
distance_m = min_distance * 111000.0
|
||||
|
||||
if best_match is not None and distance_m <= self.tolerance_meters:
|
||||
if best_match is not None:
|
||||
existing_addresses.append({
|
||||
'geometry': local_point,
|
||||
'osm_geometry': osm_gdf.iloc[best_match].geometry,
|
||||
'local_data': dict(local_row.drop('geometry')),
|
||||
'osm_data': dict(osm_gdf.iloc[best_match].drop('geometry')),
|
||||
'distance_meters': distance_m,
|
||||
'osm_geometry': osm_geoms[best_match],
|
||||
'local_data': local_props[i],
|
||||
'osm_data': osm_props[best_match],
|
||||
'distance_meters': min_distance * 111000.0,
|
||||
})
|
||||
else:
|
||||
props = dict(local_row.drop('geometry'))
|
||||
props = dict(local_props[i])
|
||||
props['status'] = 'new'
|
||||
new_addresses.append({'geometry': local_point, **props})
|
||||
|
||||
# Second pass: find OSM addresses with no local match (potentially removed)
|
||||
matched_osm = set()
|
||||
for _, local_row in local_gdf.iterrows():
|
||||
local_point = local_row.geometry
|
||||
nearby = osm_index.query(local_point.buffer(self.tolerance_deg))
|
||||
|
||||
local_num = self._normalize_multi(local_row.get('addr:housenumber', ''))
|
||||
local_street = self._normalize_street_name(str(local_row.get('addr:street', '')))
|
||||
local_unit = self._normalize_multi(local_row.get('addr:unit'))
|
||||
|
||||
for osm_idx in nearby:
|
||||
osm_row = osm_gdf.iloc[osm_idx]
|
||||
dist_m = local_point.distance(osm_row.geometry) * 111000.0
|
||||
osm_num = self._normalize_multi(osm_row.get('addr:housenumber', ''))
|
||||
osm_st = self._normalize_street_name(str(osm_row.get('addr:street', '')))
|
||||
osm_unit = self._normalize_multi(osm_row.get('addr:unit'))
|
||||
|
||||
unit_match = True
|
||||
if local_unit and osm_unit:
|
||||
unit_match = local_unit == osm_unit
|
||||
|
||||
if (dist_m <= self.tolerance_meters and
|
||||
local_num == osm_num and
|
||||
local_street == osm_st and local_street != '' and
|
||||
unit_match):
|
||||
matched_osm.add(osm_idx)
|
||||
break
|
||||
|
||||
removed_addresses = []
|
||||
for idx, osm_row in osm_gdf.iterrows():
|
||||
if idx not in matched_osm:
|
||||
props = dict(osm_row.drop('geometry'))
|
||||
for j, osm_point in enumerate(osm_geoms):
|
||||
if j not in matched_osm:
|
||||
props = dict(osm_props[j])
|
||||
# geopandas reads JSON int arrays (osm_nodes) as numpy arrays,
|
||||
# which would be written back out as a string
|
||||
for k, v in props.items():
|
||||
if isinstance(v, np.ndarray):
|
||||
props[k] = [int(x) for x in v.tolist()]
|
||||
props['status'] = 'removed'
|
||||
removed_addresses.append({'geometry': osm_row.geometry, **props})
|
||||
removed_addresses.append({'geometry': osm_point, **props})
|
||||
|
||||
self._flag_close_conflicts(new_addresses)
|
||||
renames = self._flag_possible_renames(new_addresses, removed_addresses)
|
||||
|
||||
Reference in New Issue
Block a user