diff --git a/README.md b/README.md index 0a53cfa..c8f7fba 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,19 @@ # OSM Import Tools +## Using + +### Roads + +### Addresses + +Run after Roads to reduce validation errors. Easiest to hide existing (gray) and removed (red) and just focus on what new addresses to add. + +Open in JOSM, and download the affected areas from OSM including highways with this Overpass query: + +### Paths + +Only Sumter currently has paths available. + ## Docker Quick Start ### Using Docker Compose (Recommended) @@ -47,6 +61,10 @@ docker stop osm-import-tools docker rm osm-import-tools ``` +### Deploying with Portainer + +Just push to git and set up a pull from git; Portainer can build and run the project directly. + ### Uploading to a registry ```bash docker build -t osm-import-tools . diff --git a/compare-addresses.py b/compare-addresses.py index c0c3d57..45682be 100644 --- a/compare-addresses.py +++ b/compare-addresses.py @@ -57,6 +57,20 @@ class AddressComparator: # 1 degree latitude ≈ 111,000 meters self.tolerance_deg = tolerance_meters / 111000.0 + # Load street name exceptions from YAML + exceptions_path = Path('/data/exceptions.yml') + if exceptions_path.exists(): + import yaml + with open(exceptions_path) as f: + data = yaml.safe_load(f) or {} + self._exceptions = { + item['from']: item['to'] + for item in data.get('corrections', []) + if 'from' in item and 'to' in item + } + else: + self._exceptions = {} + def _get_county_area_id(self, county: str, state: str) -> int: """Get OSM area ID for a county using Nominatim.""" search_query = f"{county} County, {state}, USA" @@ -230,12 +244,24 @@ out geom;""" # Load and process shapefile gdf = gpd.read_file(shp_file) - + + # Filter to active records only (Sumter uses LIFECYCLE field) + LIFECYCLE_FIELD = 'LIFECYCLE' + ACTIVE_VALUE = 'Current' + if LIFECYCLE_FIELD in gdf.columns: + before = len(gdf) + gdf = gdf[gdf[LIFECYCLE_FIELD] == ACTIVE_VALUE].copy().reset_index(drop=True) + filtered = before - len(gdf) + if filtered: + print(f"Filtered out {filtered} non-active addresses (LIFECYCLE != '{ACTIVE_VALUE}')") + else: + print(f"All {len(gdf)} addresses are active (LIFECYCLE == '{ACTIVE_VALUE}')") + # Convert CRS to WGS84 if needed if gdf.crs and gdf.crs != 'EPSG:4326': print(f"Converting from {gdf.crs} to EPSG:4326") gdf = gdf.to_crs('EPSG:4326') - + # Process address fields using existing logic from sumter-address-convert.py gdf = self._process_address_fields(gdf) @@ -343,6 +369,13 @@ out geom;""" address_mapping['addr:street'] = street_names + # Apply street name exceptions (e.g. county data quirks fixed for comparison/import) + if 'addr:street' in address_mapping and self._exceptions: + address_mapping['addr:street'] = [ + self._exceptions.get(name, name) if name is not None else None + for name in address_mapping['addr:street'] + ] + # City - try multiple field names city_fields = ['POST_COMM', 'PostalCity', 'CITY', 'Jurisdicti'] for field in city_fields: @@ -375,6 +408,21 @@ out geom;""" return processed_gdf + @staticmethod + def _normalize_multi(value) -> str: + """Normalize a semicolon-delimited field by sorting its parts. + + Treats '101;201;301' and '301;101;201' as equal — only real + content differences (not ordering) count as added/removed. + """ + if value is None or (isinstance(value, float) and pd.isna(value)): + return '' + s = str(value).strip() + if not s or s == 'nan': + return '' + parts = sorted(p.strip().lower() for p in s.split(';') if p.strip()) + return ';'.join(parts) + def _normalize_street_name(self, street: str) -> str: """ Normalize street names for better matching. @@ -464,8 +512,9 @@ out geom;""" distance = local_point.distance(osm_point) # Verify house number, street, and unit (if present) match - local_house_num = str(local_row.get('addr:housenumber', '')) - osm_house_num = str(osm_row.get('addr:housenumber', '')) + # _normalize_multi sorts semicolon-delimited values so order doesn't matter + local_house_num = self._normalize_multi(local_row.get('addr:housenumber', '')) + osm_house_num = self._normalize_multi(osm_row.get('addr:housenumber', '')) # Check street name match (required) - use normalization for better matching local_street = self._normalize_street_name(str(local_row.get('addr:street', ''))) @@ -473,16 +522,15 @@ out geom;""" street_match = (local_street == osm_street and local_street != '') # Check unit match - if either has a unit, both must match - local_unit = local_row.get('addr:unit') - osm_unit = osm_row.get('addr:unit') + local_unit_norm = self._normalize_multi(local_row.get('addr:unit')) + osm_unit_norm = self._normalize_multi(osm_row.get('addr:unit')) - # Determine if each side has a unit - local_has_unit = local_unit is not None and pd.notna(local_unit) and str(local_unit).strip() != '' - osm_has_unit = osm_unit is not None and pd.notna(osm_unit) and str(osm_unit).strip() != '' + local_has_unit = bool(local_unit_norm) + osm_has_unit = bool(osm_unit_norm) if local_has_unit and osm_has_unit: - # Both have units - they must match - unit_match = (str(local_unit).strip().lower() == str(osm_unit).strip().lower()) + # Both have units - they must match (order-insensitive) + unit_match = (local_unit_norm == osm_unit_norm) elif local_has_unit or osm_has_unit: # One has unit, other doesn't - no match unit_match = False @@ -538,21 +586,20 @@ out geom;""" distance_meters = local_point.distance(osm_point) * 111000.0 # Verify house number, street, and unit (if present) match - local_house_num = str(local_row.get('addr:housenumber', '')) - osm_house_num = str(osm_row.get('addr:housenumber', '')) + local_house_num = self._normalize_multi(local_row.get('addr:housenumber', '')) + osm_house_num = self._normalize_multi(osm_row.get('addr:housenumber', '')) # Check street name match (required) - use normalization for better matching local_street = self._normalize_street_name(str(local_row.get('addr:street', ''))) osm_street = self._normalize_street_name(str(osm_row.get('addr:street', ''))) street_match = (local_street == osm_street and local_street != '') - # Check unit match (only if both have units specified) - local_unit = local_row.get('addr:unit') - osm_unit = osm_row.get('addr:unit') + # Check unit match (order-insensitive via _normalize_multi) + local_unit_norm = self._normalize_multi(local_row.get('addr:unit')) + osm_unit_norm = self._normalize_multi(osm_row.get('addr:unit')) unit_match = True - if local_unit is not None and pd.notna(local_unit) and osm_unit is not None and pd.notna(osm_unit): - # Both have units - they must match - unit_match = (str(local_unit).strip().lower() == str(osm_unit).strip().lower()) + if local_unit_norm and osm_unit_norm: + unit_match = (local_unit_norm == osm_unit_norm) if (distance_meters <= self.tolerance_meters and local_house_num == osm_house_num and diff --git a/diff-highways.py b/diff-highways.py index 9886815..34c0997 100644 --- a/diff-highways.py +++ b/diff-highways.py @@ -13,15 +13,9 @@ TODO: - ignore points outside of lines - put properties properly on removed roads, so they're visible in JOSM - handle polygons properly (on previous geojson step?) for circular roads -- ignore roads that aren't LIFECYCLE ACTV or Active - include OneWay=Y - handle C 44a -> County Road 44A -- handle Tpke -> Turnpike - handle Trce -> Trace/Terrace? -- handle Cor -> Corner -- handle Obrien -> O'Brien -- handle Oday -> O'Day -- Ohara -> O'Hara """ import json @@ -461,6 +455,21 @@ class RoadComparator: # Filter unnamed features from file1 (OSM data) if exclude_unnamed is set gdf1 = self.load_geojson(file1_path, filter_unnamed=self.exclude_unnamed) gdf2 = self.load_geojson(file2_path) + + # Filter county data (gdf2) to active/current roads only. + # Sumter uses LIFECYCLE='ACTV'. Lake has no lifecycle field — skip silently. + LIFECYCLE_FIELD = 'LIFECYCLE' + ACTIVE_VALUES = {'ACTV', 'Active', 'ACTIVE', 'Current', 'CURRENT'} + if LIFECYCLE_FIELD in gdf2.columns: + before = len(gdf2) + gdf2 = gdf2[gdf2[LIFECYCLE_FIELD].isin(ACTIVE_VALUES)].copy().reset_index(drop=True) + filtered = before - len(gdf2) + if filtered: + print(f"Filtered out {filtered} non-active county roads (LIFECYCLE not in {ACTIVE_VALUES})") + else: + print(f"All {len(gdf2)} county roads are active (LIFECYCLE={gdf2[LIFECYCLE_FIELD].iloc[0] if len(gdf2) else 'n/a'})") + else: + print(f"Note: county data has no {LIFECYCLE_FIELD!r} field — no lifecycle filtering applied") # Ensure both are in the same CRS if gdf1.crs != gdf2.crs: diff --git a/qgis-functions.py b/qgis-functions.py index 68a4ad3..838215f 100644 --- a/qgis-functions.py +++ b/qgis-functions.py @@ -8,11 +8,18 @@ import re # or >1 suffix-letters, like 12th Street or 243rd Ave. # +_POST_TITLE_CORRECTIONS = [ + (r'\bUs\b', 'US'), # "US 441" not "Us 441" +] + def title(s): - return re.sub( + result = re.sub( r"[A-Za-z0-9]+('[A-Za-z0-9]+)?", lambda word: word.group(0).capitalize(), s) + for pattern, replacement in _POST_TITLE_CORRECTIONS: + result = re.sub(pattern, replacement, result) + return result # @qgsfunction(args='auto', group='Custom', referenced_columns=[]) def getstreetfromaddress(value1, feature, parent): @@ -103,6 +110,8 @@ def formatstreetname(name): if nameUp == "NW": return "Northwest" # Names + if nameUp == "FLORIDAS": + return "Florida's" if nameUp == "MACLEAY": return "MacLeay" if nameUp == "MCCLAINE": @@ -218,6 +227,8 @@ def formatstreetname(name): return "Mount" if nameUp == "MTN": return "Mountain" + if nameUp == "COR": + return "Corner" if nameUp == "PARK": return "Park" if nameUp == "PASS": @@ -248,8 +259,12 @@ def formatstreetname(name): return "Street" if nameUp == "TER": return "Terrace" + if nameUp == "TPKE": + return "Turnpike" if nameUp == "TR": return "Trail" + if nameUp == "TRCE": + return "Terrace" if nameUp == "TRL": return "Trail" if nameUp == "VW": diff --git a/requirements.txt b/requirements.txt index 1c60d02..a8f96c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ Flask==3.0.0 +pyyaml>=6.0 numpy<2.0.0 geopandas>=0.14.0 pandas>=2.1.0 diff --git a/web/server.py b/web/server.py index a9a8948..215e131 100644 --- a/web/server.py +++ b/web/server.py @@ -8,6 +8,7 @@ import os import threading import json from datetime import datetime +import yaml app = Flask(__name__, static_folder='static', template_folder='templates') @@ -295,5 +296,64 @@ def serve_data(filename): """Serve GeoJSON files""" return send_from_directory('/data', filename) +EXCEPTIONS_FILE = '/data/exceptions.yml' + +DEFAULT_EXCEPTIONS = [ + {'from': "D Angelo Lane", 'to': "D'Angelo Lane"}, + {'from': "Lajolla Circle", 'to': "La Jolla Circle"}, + {'from': "Gardena Court", 'to': "Gardenia Court"}, + {'from': "Glenmont Court", 'to': "Glenmount Court"}, + {'from': "Pawleys Island Path", 'to': "Pawley's Island Path"}, + {'from': "Oday Street", 'to': "O'Day Street"}, + {'from': "Obrien Place", 'to': "O'Brien Place"}, + {'from': "Ohara Court", 'to': "O'Hara Court"}, +] + +def _ensure_exceptions_file(): + if not os.path.exists(EXCEPTIONS_FILE): + os.makedirs(os.path.dirname(EXCEPTIONS_FILE), exist_ok=True) + with open(EXCEPTIONS_FILE, 'w') as f: + yaml.dump({'corrections': DEFAULT_EXCEPTIONS}, f, + default_flow_style=False, allow_unicode=True) + +_ensure_exceptions_file() + + +@app.route('/exceptions') +def exceptions_page(): + return render_template('exceptions.html') + + +@app.route('/api/exceptions', methods=['GET']) +def get_exceptions(): + try: + with open(EXCEPTIONS_FILE) as f: + data = yaml.safe_load(f) or {} + return jsonify({'corrections': data.get('corrections', [])}) + except Exception as e: + return jsonify({'error': str(e)}), 500 + + +@app.route('/api/exceptions', methods=['POST']) +def save_exceptions(): + data = request.json + corrections = data.get('corrections', []) + + # Validate + for item in corrections: + if not isinstance(item.get('from'), str) or not item['from'].strip(): + return jsonify({'error': 'Each correction must have a non-empty "from" field'}), 400 + if not isinstance(item.get('to'), str) or not item['to'].strip(): + return jsonify({'error': 'Each correction must have a non-empty "to" field'}), 400 + + # Atomic write + tmp = EXCEPTIONS_FILE + '.tmp' + with open(tmp, 'w') as f: + yaml.dump({'corrections': corrections}, f, + default_flow_style=False, allow_unicode=True) + os.replace(tmp, EXCEPTIONS_FILE) + return jsonify({'success': True}) + + if __name__ == '__main__': app.run(host='0.0.0.0', port=5000, debug=True) diff --git a/web/static/map.js b/web/static/map.js index 82ac01b..e130c15 100644 --- a/web/static/map.js +++ b/web/static/map.js @@ -352,10 +352,13 @@ function countyMarkerStyle(feature) { // Filter function for OSM features function shouldShowOsmFeature(feature) { const props = feature.properties || {}; - const isService = props.highway === 'service'; + const isService = props.highway === 'service' || props.highway === 'track'; + const isUnclassified = props.highway === 'unclassified'; const hideService = document.getElementById('hideService').checked; + const hideUnclassified = document.getElementById('hideUnclassified').checked; if (isService && hideService) return false; + if (isUnclassified && hideUnclassified) return false; return true; } @@ -424,18 +427,21 @@ function createOsmLayer() { function shouldShowFeature(feature) { const props = feature.properties || {}; const isRemoved = props.removed === true || props.removed === 'True'; - const isService = props.highway === 'service'; + const isService = props.highway === 'service' || props.highway === 'track'; + const isUnclassified = props.highway === 'unclassified'; const showAdded = document.getElementById('showAdded').checked; const showRemoved = document.getElementById('showRemoved').checked; const hideService = document.getElementById('hideService').checked; + const hideUnclassified = document.getElementById('hideUnclassified').checked; // Check removed/added filter if (isRemoved && !showRemoved) return false; if (!isRemoved && !showAdded) return false; - // Check service filter + // Check service/track and unclassified filters if (isService && hideService) return false; + if (isUnclassified && hideUnclassified) return false; return true; } @@ -504,10 +510,13 @@ function createDiffLayer() { // Filter function for county features function shouldShowCountyFeature(feature) { const props = feature.properties || {}; - const isService = props.highway === 'service'; + const isService = props.highway === 'service' || props.highway === 'track'; + const isUnclassified = props.highway === 'unclassified'; const hideService = document.getElementById('hideService').checked; + const hideUnclassified = document.getElementById('hideUnclassified').checked; if (isService && hideService) return false; + if (isUnclassified && hideUnclassified) return false; return true; } @@ -694,7 +703,20 @@ function showMultiFeaturePopup(latlng) { 'addr:postcode': 4, 'addr:state': 5, 'name': 10, - 'highway': 11 + 'highway': 11, + // County road fields + 'Lifecycle': 20, + 'LIFECYCLE': 20, + 'FullStreetN': 21, + 'FullName': 21, + 'StreetClass': 22, + 'STRCLASS': 22, + 'SpeedLimit': 23, + 'SPEEDLIMIT': 23, + 'LeftCity': 24, + 'RightCity': 25, + 'LeftZip': 26, + 'RightZip': 27, }; const sortedKeys = Array.from(allKeys).sort((a, b) => { @@ -1201,6 +1223,17 @@ document.addEventListener('DOMContentLoaded', function() { createCountyLayer(); }); + document.getElementById('hideUnclassified').addEventListener('change', function() { + createDiffLayer(); + createOsmLayer(); + createCountyLayer(); + }); + + // Click on empty map area closes any open popup and clears selection + map.on('click', function() { + clearSelection(); + }); + // Load button document.getElementById('loadButton').addEventListener('click', function() { loadFiles(); diff --git a/web/templates/exceptions.html b/web/templates/exceptions.html new file mode 100644 index 0000000..43797e2 --- /dev/null +++ b/web/templates/exceptions.html @@ -0,0 +1,323 @@ + + + + + + Street Exceptions — OSM Import Tools + + + +
+ +
+

OSM Import Tools

+
+ ← Back +
+ Street Exceptions + Open Map Viewer +
+ +
+
+
+
+

Street Name Exceptions

+

Applied to county address data after formatting, before comparison and import. Match is exact (case-sensitive).

+
+
+ +
+ + + + + + + + + + + + +
County data (from)Corrected name (to)
Loading...
+ +
+ + +
+
+
+ +
+ + + + diff --git a/web/templates/index.html b/web/templates/index.html index 6dd1c89..42c24df 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -330,6 +330,7 @@
+ Street Exceptions Open Map Viewer diff --git a/web/templates/map.html b/web/templates/map.html index 000af77..e5863bf 100644 --- a/web/templates/map.html +++ b/web/templates/map.html @@ -294,7 +294,7 @@
- OSM Roads (Gray) + OSM Layer (Gray)
@@ -313,7 +313,11 @@ +

Selection Mode