From 7ce73f19c55b637f8a5ab6d5f64ac691df40800a Mon Sep 17 00:00:00 2001 From: zyphlar Date: Fri, 28 Aug 2026 14:23:26 -0700 Subject: [PATCH] Create new project design plans --- Dockerfile | 1 + README.md | 2 + compare-addresses.py | 17 +- convert-addresses.py | 34 +- docker-compose.override.yml | 2 + download-overpass.py | 84 +-- generate-city-report.py | 104 ++++ map-checker/README.md | 70 +++ map-checker/check-boundary.py | 163 ++++++ map-checker/check.py | 209 ++++++++ map-checker/reference.yml | 32 ++ web/server.py | 129 +++++ web/static/boundary-tracer.js | 478 +++++++++++++++++ web/static/map-ui.css | 207 ++++++++ web/static/map-ui.js | 41 ++ web/static/map.js | 35 +- web/static/report-map.js | 597 ++++++++++++++++++++++ web/templates/boundary-tracer.html | 173 +++++++ web/templates/city-boundaries-report.html | 243 +++++++++ web/templates/index.html | 9 + web/templates/map.html | 200 +------- web/templates/reports.html | 115 +++++ 22 files changed, 2682 insertions(+), 263 deletions(-) create mode 100644 generate-city-report.py create mode 100644 map-checker/README.md create mode 100644 map-checker/check-boundary.py create mode 100644 map-checker/check.py create mode 100644 map-checker/reference.yml create mode 100644 web/static/boundary-tracer.js create mode 100644 web/static/map-ui.css create mode 100644 web/static/map-ui.js create mode 100644 web/static/report-map.js create mode 100644 web/templates/boundary-tracer.html create mode 100644 web/templates/city-boundaries-report.html create mode 100644 web/templates/reports.html diff --git a/Dockerfile b/Dockerfile index d8a0057..065d1de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ RUN pip install --no-cache-dir -r requirements.txt # Copy only necessary Python scripts and web files COPY *.py ./ COPY web/ ./web/ +COPY map-checker/ ./map-checker/ # Create /data directory (will be mounted as volume) RUN mkdir -p /data diff --git a/README.md b/README.md index 5ef2610..f300c43 100644 --- a/README.md +++ b/README.md @@ -219,3 +219,5 @@ addr:state=FL addr:street=Hideaway Circle - Sr is SR (State Route) - Pky is Parkway +- The county sometimes lists garages or other units in the street name like "Vista Hills Street Garage 2" or "... Gs2" / "... Gs 2" which needs to be properly split out into street and unit +- Roads/lines need to be more clickable on the web UI, it's hard to click a ~3px element. \ No newline at end of file diff --git a/compare-addresses.py b/compare-addresses.py index 88ef42e..2dfe4af 100644 --- a/compare-addresses.py +++ b/compare-addresses.py @@ -155,6 +155,7 @@ class AddressComparator: if best_match is not None and distance_m <= self.tolerance_meters: existing_addresses.append({ 'geometry': local_point, + 'osm_geometry': osm_gdf.iloc[best_match].geometry, 'local_data': dict(local_row.drop('geometry')), 'osm_data': dict(osm_gdf.iloc[best_match].drop('geometry')), 'distance_meters': distance_m, @@ -218,8 +219,22 @@ class AddressComparator: print(f"Saved {len(removed_addresses)} potentially removed to {output_dir}/addresses-potentially-removed.geojson") if existing_addresses: + # Keeps both county city fields (postal community and legal + # city/jurisdiction - they can differ) plus OSM's addr:city, and + # the OSM point's coordinates, so the "Generate City Boundaries + # Report" step can build its report from this file alone without + # repeating the spatial match. existing_simple = [ - {'geometry': a['geometry'], 'distance_meters': a['distance_meters'], 'status': 'existing'} + { + 'geometry': a.get('osm_geometry') or a['geometry'], + 'distance_meters': a['distance_meters'], + 'addr:housenumber': a['local_data'].get('addr:housenumber'), + 'addr:street': a['local_data'].get('addr:street'), + 'addr:unit': a['local_data'].get('addr:unit'), + 'county_postal_community': a['local_data'].get('county_postal_community'), + 'county_city': a['local_data'].get('county_city'), + 'osm_city': a['osm_data'].get('addr:city'), + } for a in existing_addresses ] gpd.GeoDataFrame(existing_simple).to_file( diff --git a/convert-addresses.py b/convert-addresses.py index 11a6d2a..cacb5c6 100644 --- a/convert-addresses.py +++ b/convert-addresses.py @@ -133,6 +133,35 @@ def process_address_fields(gdf, exceptions): ] break + # Postal community (USPS mailing city) and city/jurisdiction (legal + # municipal boundary) are tracked separately from addr:city above - + # they're often the same value, but not always (e.g. a parcel legally + # inside Leesburg or Sumterville whose mail is addressed to "The + # Villages"). Kept for the City Boundaries report; addr:city above is + # still the single value actually used for the OSM import itself. + for field in ['POST_COMM', 'PostalCity']: + if field in processed.columns: + mapping['county_postal_community'] = [ + qgis_functions.title(str(v)) if pd.notna(v) else None + for v in processed[field] + ] + break + + def _clean_jurisdiction(v): + if pd.isna(v): + return None + s = str(v).strip() + if s.upper().startswith('UNINCORPORATED '): + s = s[len('UNINCORPORATED '):] + elif s.upper() == 'UNINCORPORATED': + return None + return qgis_functions.title(s) if s else None + + for field in ['CITY', 'Jurisdicti']: + if field in processed.columns: + mapping['county_city'] = [_clean_jurisdiction(v) for v in processed[field]] + break + # Apply street name exceptions (some are scoped to a specific city) if 'addr:street' in mapping and exceptions: mapping['addr:street'] = apply_street_exceptions( @@ -201,7 +230,10 @@ def convert(zip_path, output_path): gdf = gdf[gdf['addr:housenumber'].notna()].copy() osm_fields = ['addr:housenumber', 'addr:unit', 'addr:street', - 'addr:city', 'addr:postcode', 'addr:state'] + 'addr:city', 'addr:postcode', 'addr:state', + # Not real OSM tags - carried through for the City + # Boundaries report only, not part of the OSM import. + 'county_postal_community', 'county_city'] keep = [f for f in osm_fields if f in gdf.columns] gdf = gdf[keep + ['geometry']] diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 38cc387..73a288f 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -13,3 +13,5 @@ services: - ./shp-to-geojson.py:/app/shp-to-geojson.py - ./convert-addresses.py:/app/convert-addresses.py - ./compare-addresses.py:/app/compare-addresses.py + - ./generate-city-report.py:/app/generate-city-report.py + - ./map-checker:/app/map-checker diff --git a/download-overpass.py b/download-overpass.py index 551b7e2..83e2848 100644 --- a/download-overpass.py +++ b/download-overpass.py @@ -82,37 +82,53 @@ area(id:{area_id})->.searchArea;""" return query -def query_overpass(query): - """Send query to Overpass API and return JSON response, with retries.""" - url = "https://overpass-api.de/api/interpreter" - data = urllib.parse.urlencode({"data": query}).encode("utf-8") - req = urllib.request.Request( - url, - data=data, - headers={"Accept": "*/*", "User-Agent": "TheVillagesImport/1.0"}, - ) +# Public Overpass API mirrors, tried in order when the primary is overloaded. +OVERPASS_ENDPOINTS = [ + "https://overpass-api.de/api/interpreter", + "https://overpass.kumi.systems/api/interpreter", + "https://overpass.private.coffee/api/interpreter", +] - max_attempts = 3 - for attempt in range(1, max_attempts + 1): - try: - with urllib.request.urlopen(req, timeout=300) as response: - return json.loads(response.read().decode("utf-8")) - except urllib.error.HTTPError as e: + +def query_overpass(query, endpoints=None): + """Send query to Overpass API and return JSON response. + + Tries each endpoint in turn (with a couple of retries per endpoint on + 429/504) before falling through to the next mirror. Overpass instances + share the underlying database, so any mirror returns equivalent data. + """ + endpoints = endpoints or OVERPASS_ENDPOINTS + data = urllib.parse.urlencode({"data": query}).encode("utf-8") + + attempts_per_endpoint = 2 + for endpoint in endpoints: + req = urllib.request.Request( + endpoint, + data=data, + headers={"Accept": "*/*", "User-Agent": "TheVillagesImport/1.0"}, + ) + for attempt in range(1, attempts_per_endpoint + 1): try: - error_body = e.read().decode("utf-8") - except Exception: - error_body = "(unreadable)" - print(f"HTTP Error {e.code} on attempt {attempt}: {e.reason}", file=sys.stderr) - print(f"Error response body: {error_body}", file=sys.stderr) - if attempt < max_attempts and e.code in (429, 504): - wait = 30 * attempt - print(f"Retrying in {wait}s...", file=sys.stderr) - time.sleep(wait) - else: - sys.exit(1) - except Exception as e: - print(f"Error querying Overpass API: {e}", file=sys.stderr) - sys.exit(1) + print(f"Querying {endpoint} (attempt {attempt})...", file=sys.stderr) + with urllib.request.urlopen(req, timeout=300) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as e: + try: + error_body = e.read().decode("utf-8") + except Exception: + error_body = "(unreadable)" + print(f"HTTP Error {e.code} on {endpoint} attempt {attempt}: {e.reason}", file=sys.stderr) + print(f"Error response body: {error_body}", file=sys.stderr) + if attempt < attempts_per_endpoint and e.code in (429, 504): + wait = 30 * attempt + print(f"Retrying in {wait}s...", file=sys.stderr) + time.sleep(wait) + except Exception as e: + print(f"Error querying {endpoint}: {e}", file=sys.stderr) + break # non-HTTP error (timeout, connection refused): skip to next mirror + + print("All Overpass endpoints failed.", file=sys.stderr) + sys.exit(1) def convert_to_geojson(overpass_data): @@ -200,7 +216,13 @@ def main(): default="highways", help="Type of data to download (default: highways)" ) - + parser.add_argument( + "--endpoint", "-e", + action="append", + help="Overpass API endpoint to use (repeatable; tried in order). " + f"Default: {', '.join(OVERPASS_ENDPOINTS)}" + ) + args = parser.parse_args() # Create output directory if it doesn't exist @@ -212,7 +234,7 @@ def main(): # Build and execute query query = build_overpass_query(args.county, args.state, args.type) print(f"Query: {query}") - overpass_data = query_overpass(query) + overpass_data = query_overpass(query, endpoints=args.endpoint) # Convert to GeoJSON geojson = convert_to_geojson(overpass_data) diff --git a/generate-city-report.py b/generate-city-report.py new file mode 100644 index 0000000..7b05619 --- /dev/null +++ b/generate-city-report.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +Generate the City Boundaries report: addr:city agreement between the county +source and OpenStreetMap for already-matched addresses. + +Reads addresses-existing.geojson (produced by compare-addresses.py, which +already carries county_postal_community/county_city/osm_city for each +matched pair) and classifies each point against OSM twice - once per county +field, since the postal community (USPS mailing city) and the legal +city/jurisdiction can disagree with each other, not just with OSM. No +spatial matching happens here, so this is fast/cheap to re-run on its own +without repeating the "Diff vs OSM" step. + +Usage: + python generate-city-report.py \ + --existing-file /data/lake/addresses-existing.geojson \ + --output-dir /data/lake +""" + +import argparse +import sys +import warnings +from pathlib import Path + +import geopandas as gpd +import pandas as pd + +warnings.filterwarnings('ignore') + + +def normalize_city(value) -> str: + if value is None or (isinstance(value, float) and pd.isna(value)): + return '' + s = str(value).strip() + return '' if s in ('', 'nan', 'None') else s + + +def agreement_status(county_city, osm_city) -> str: + c = normalize_city(county_city) + o = normalize_city(osm_city) + if not c and not o: + return 'both_missing' + if not c: + return 'osm_only' + if not o: + return 'county_only' + return 'match' if c.lower() == o.lower() else 'mismatch' + + +def main(): + parser = argparse.ArgumentParser( + description="Generate the city-name agreement report from already-matched addresses", + epilog="Run compare-addresses.py first to produce addresses-existing.geojson." + ) + parser.add_argument('--existing-file', required=True, + help='addresses-existing.geojson from compare-addresses.py') + parser.add_argument('--output-dir', '-o', required=True, + help='Output directory for city-comparison.geojson') + args = parser.parse_args() + + existing_file = Path(args.existing_file) + output_dir = Path(args.output_dir) + + if not existing_file.exists(): + print(f"Error: {existing_file} not found.", file=sys.stderr) + print("Run 'Diff vs OSM' for addresses first.", file=sys.stderr) + return 1 + + gdf = gpd.read_file(existing_file) + + required = ['county_postal_community', 'county_city', 'osm_city'] + missing = [c for c in required if c not in gdf.columns] + if missing: + print(f"Error: {existing_file} is missing column(s): {', '.join(missing)}", file=sys.stderr) + print("This file predates the postal-community/city split - rerun " + "'Convert to GeoJSON' and 'Diff vs OSM' for this county " + "(convert-addresses.py needs to regenerate county-addresses.geojson " + "with the current field set) before generating the report.", file=sys.stderr) + return 1 + + gdf['status_postal_community'] = [ + agreement_status(c, o) + for c, o in zip(gdf['county_postal_community'], gdf['osm_city']) + ] + gdf['status_city'] = [ + agreement_status(c, o) + for c, o in zip(gdf['county_city'], gdf['osm_city']) + ] + + output_dir.mkdir(parents=True, exist_ok=True) + out_path = output_dir / "city-comparison.geojson" + gdf.to_file(out_path, driver='GeoJSON') + + print(f"Saved {len(gdf)} city comparisons to {out_path}") + for label, col in [('postal community', 'status_postal_community'), ('city', 'status_city')]: + print(f" vs {label}:") + for status, n in gdf[col].value_counts().items(): + print(f" {status}: {n}") + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/map-checker/README.md b/map-checker/README.md new file mode 100644 index 0000000..4631714 --- /dev/null +++ b/map-checker/README.md @@ -0,0 +1,70 @@ +# map-checker + +Cross-references our Sumter/Lake county address data against a reference map +(e.g. [thevillages.com/map](https://www.thevillages.com/map/)) to catch city +boundary or misnaming issues (e.g. addresses that are really in The Villages +but our data has as Lady Lake, Sumterville, Leesburg, etc.). + +## Why this is manual, not automated + +thevillages.com's Terms of Use explicitly prohibit automated access: + +> "You must not use any robot, spider or other automatic device, process or +> means to access the Website for any purpose, including monitoring or +> copying any of the material on the Website." + +So this subproject never fetches, scrapes, or otherwise programmatically +touches their site. Every input here comes from a human looking at the +official map in their own browser and either tracing a boundary or recording +an observation by hand. + +## Workflow 1: hand-traced boundary (recommended) + +Recording individual addresses one at a time doesn't scale to tens of +thousands of records. Instead, trace the boundary **once** and check every +address against it in one pass. + +1. Open `https://www.thevillages.com/map/` (or another reference map) in one + browser tab. +2. Open **`/map-checker`** in the app (the "Boundary Tracer" link on the main + page) in another tab. +3. Click "New polygon", then click the map to place vertices tracing the + boundary you see on the reference map. The Villages is disjoint across + three counties, so you'll likely want several separate polygons - just + name them all the same (e.g. "The Villages") and add another with "New + polygon" once you finish the current one. +4. Click "Finish polygon", give it a name, repeat for each disjoint piece. +5. Click "Save to server" (requires login) - this writes + `/data/map-checker/boundary.geojson`. "Download GeoJSON" also works if you + just want a local copy. +6. Run the point-in-polygon check: + + ```bash + docker compose exec web python3 map-checker/check-boundary.py --county lake --county sumter + ``` + + This flags: + - addresses **inside** the traced boundary whose county/OSM city fields + don't say "The Villages" (probably under-tagged), and + - addresses **outside** it that do say "The Villages" anyway (probably + over-tagged or the boundary needs another pass). + + Pass `--expect-city "Some Other City"` to check a different boundary/name, + and `--output flagged.geojson` to save the flagged points for review in + the main map viewer. + +## Workflow 2: individual spot-checks + +For one-off verification of a specific address (not a bulk boundary check), +[`reference.yml`](reference.yml) + `check.py` still works the same way as +before: hand-record what the reference map says for a specific street/area, +then run `python check.py` to compare it against our data. See the comments +at the top of `reference.yml` for the entry format. + +## Scope + +`check-boundary.py` is only as accurate as the hand-traced polygon - it's a +spot-check/audit tool, not a source of truth. For a systematic (but coarser) +look at where the county's own postal-community and city/jurisdiction fields +disagree with each other and with OSM, see the "City Boundaries" report in +the main app (`/reports/city-boundaries`). diff --git a/map-checker/check-boundary.py b/map-checker/check-boundary.py new file mode 100644 index 0000000..da04671 --- /dev/null +++ b/map-checker/check-boundary.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +Point-in-polygon check: for a hand-traced boundary (see the Boundary Tracer +tool at /map-checker, or draw manually and save as boundary.geojson), +flag every county/OSM address that disagrees with it - either inside the +boundary but not labeled with the expected city, or outside it but labeled +with the expected city anyway. + +The boundary itself is drawn by a human looking at a reference map in their +own browser (e.g. thevillages.com/map) - nothing here fetches from any +third-party site, only our own /data/{county}/*.geojson files. + +Usage: + python check-boundary.py --county lake --county sumter + python check-boundary.py --county lake --expect-city "The Villages" --output flagged.geojson +""" + +import argparse +import json +import sys +from pathlib import Path + +from shapely.geometry import shape, Point +from shapely.ops import unary_union +import warnings + +warnings.filterwarnings('ignore') + + +def normalize_city(value): + if value is None: + return '' + s = str(value).strip() + return '' if s in ('', 'nan', 'None') else s + + +def load_boundary(path): + with open(path) as f: + data = json.load(f) + polys = [] + for feat in data.get('features', []): + name = (feat.get('properties') or {}).get('name', 'boundary') + polys.append((name, shape(feat['geometry']))) + return polys + + +def load_addresses(data_dir, county): + county_dir = Path(data_dir) / county + # Prefer city-comparison.geojson: has postal community, city, AND osm_city + # together for matched addresses. Falls back to county-addresses.geojson + # (county-only, addr:city, no OSM side) if the diff step hasn't run yet. + for filename in ('city-comparison.geojson', 'county-addresses.geojson'): + path = county_dir / filename + if path.exists(): + with open(path) as f: + return json.load(f), filename + return None, None + + +def city_values(props): + values = [ + normalize_city(props.get('county_postal_community')), + normalize_city(props.get('county_city')), + normalize_city(props.get('osm_city')), + normalize_city(props.get('addr:city')), + ] + return [v for v in values if v] + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--boundary', default='/data/map-checker/boundary.geojson', + help='Path to the traced boundary GeoJSON (default: the file saved by ' + 'the Boundary Tracer web tool at /map-checker)') + parser.add_argument('--data-dir', default='/data') + parser.add_argument('--county', action='append', + help='County id to check (repeatable). Default: lake, sumter') + parser.add_argument('--expect-city', default='The Villages', + help='City name addresses inside the boundary should have (default: "The Villages")') + parser.add_argument('--output', help='Optional path to save flagged addresses as GeoJSON') + args = parser.parse_args() + + boundary_path = Path(args.boundary) + if not boundary_path.exists(): + print(f"Error: {boundary_path} not found.", file=sys.stderr) + print("Trace a boundary first via the Boundary Tracer tool (/map-checker), " + "or save one to that path by hand.", file=sys.stderr) + return 1 + + polys = load_boundary(boundary_path) + if not polys: + print("Error: boundary file has no polygons.", file=sys.stderr) + return 1 + combined = unary_union([g for _, g in polys]) + + counties = args.county or ['lake', 'sumter'] + expect = args.expect_city.strip().lower() + + total = inside_count = inside_flagged = outside_flagged = 0 + flagged_features = [] + + print(f"Boundary: {', '.join(sorted(set(name for name, _ in polys)))} " + f"({len(polys)} polygon(s))") + + for county in counties: + fc, source_file = load_addresses(args.data_dir, county) + if fc is None: + print(f" {county}: no data found, skipping.") + continue + + county_total = county_inside = county_inside_flagged = county_outside_flagged = 0 + + for feat in fc['features']: + coords = feat['geometry']['coordinates'] + pt = Point(coords[0], coords[1]) + is_inside = combined.contains(pt) + props = feat.get('properties', {}) + values = city_values(props) + says_expected = any(v.lower() == expect for v in values) + + county_total += 1 + if is_inside: + county_inside += 1 + if values and not says_expected: + county_inside_flagged += 1 + flagged_features.append({ + **feat, + 'properties': {**props, 'county': county, 'flag': 'inside_but_not_labeled'}, + }) + elif says_expected: + county_outside_flagged += 1 + flagged_features.append({ + **feat, + 'properties': {**props, 'county': county, 'flag': 'outside_but_labeled'}, + }) + + print(f" {county} ({source_file}): {county_total} addresses, " + f"{county_inside} inside boundary " + f"({county_inside_flagged} not labeled '{args.expect_city}'), " + f"{county_outside_flagged} outside boundary but labeled '{args.expect_city}'") + + total += county_total + inside_count += county_inside + inside_flagged += county_inside_flagged + outside_flagged += county_outside_flagged + + print(f"\n{'=' * 60}") + print(f"Total: {total} addresses checked, {inside_count} inside the boundary.") + print(f" Inside but not labeled '{args.expect_city}': {inside_flagged}") + print(f" Outside but labeled '{args.expect_city}': {outside_flagged}") + + if args.output: + out = {'type': 'FeatureCollection', 'features': flagged_features} + with open(args.output, 'w') as f: + json.dump(out, f) + print(f"\nSaved {len(flagged_features)} flagged addresses to {args.output}") + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/map-checker/check.py b/map-checker/check.py new file mode 100644 index 0000000..bb7f847 --- /dev/null +++ b/map-checker/check.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +""" +map-checker: compare hand-recorded observations from thevillages.com/map +(see reference.yml) against our county/OSM address data. + +This is a deliberately manual workflow. thevillages.com's Terms of Use +prohibit automated/bot access to their site, so nothing here fetches from +them - reference.yml is filled in BY HAND after a human looks at the +official map in their own browser. This script only reads OUR OWN data +(county-addresses.geojson / city-comparison.geojson, already produced by +convert-addresses.py and generate-city-report.py) and checks it against +those manually recorded entries. + +Usage: + python check.py [--reference reference.yml] [--data-dir /data] [--radius 200] +""" + +import argparse +import json +import math +import re +import sys +from pathlib import Path + +import yaml + +DEFAULT_RADIUS_METERS = 200.0 + + +def normalize_city(value): + if not value: + return '' + s = str(value).strip() + return '' if s in ('', 'nan', 'None') else s + + +def normalize_street(value): + if not value: + return '' + s = str(value).strip().lower() + s = re.sub(r'\bstate road\b', 'sr', s) + s = re.sub(r'\bstate route\b', 'sr', s) + s = re.sub(r'\bcounty road\b', 'cr', s) + s = re.sub(r'\s+', ' ', s).strip() + return s + + +def haversine_meters(lat1, lon1, lat2, lon2): + r = 6371000.0 + p1, p2 = math.radians(lat1), math.radians(lat2) + dp = math.radians(lat2 - lat1) + dl = math.radians(lon2 - lon1) + a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2 + return 2 * r * math.asin(math.sqrt(a)) + + +def load_geojson(path): + if not path.exists(): + return None + with open(path) as f: + return json.load(f) + + +def find_matches(entry, county_features, comparison_features, radius_m): + """Return a list of matched feature-property dicts for one reference entry. + + Prefers city-comparison.geojson (has county_postal_community, county_city, + AND osm_city together) when available; falls back to the raw + county-addresses.geojson (county-only, no OSM side) otherwise. + """ + source = comparison_features if comparison_features is not None else county_features + if source is None: + return [] + + street = entry.get('street') + housenumber = entry.get('housenumber') + lat, lon = entry.get('lat'), entry.get('lon') + + matches = [] + for feature in source['features']: + props = feature.get('properties', {}) + + if lat is not None and lon is not None: + coords = feature['geometry']['coordinates'] + dist = haversine_meters(lat, lon, coords[1], coords[0]) + if dist > radius_m: + continue + elif street: + if normalize_street(props.get('addr:street')) != normalize_street(street): + continue + if housenumber is not None: + try: + if int(props.get('addr:housenumber')) != int(housenumber): + continue + except (TypeError, ValueError): + continue + else: + continue + + matches.append(props) + + return matches + + +def check_entry(entry, index, county_features, comparison_features, radius_m): + label = entry.get('street') or f"({entry.get('lat')}, {entry.get('lon')})" + official_city = normalize_city(entry.get('official_city')) + matches = find_matches(entry, county_features, comparison_features, radius_m) + + print(f"\n[{index}] {label}" + + (f" #{entry['housenumber']}" if entry.get('housenumber') else '') + + f" ({entry.get('county', '?')})") + if entry.get('official_village'): + print(f" Official village: {entry['official_village']}") + print(f" Official city (recorded): {official_city or '(none recorded)'}") + + if not matches: + print(" No matching addresses found in our data - can't check.") + return {'checked': True, 'matched': False, 'mismatches': 0} + + # A single street address (e.g. a shopping plaza or apartment building) + # can match many unit rows that all carry identical city values - group + # those together instead of printing one line per unit. + groups = {} # (postal_community, city, osm_city) -> {'count', 'addr'} + for props in matches: + addr = ' '.join(str(v) for v in + [props.get('addr:housenumber'), props.get('addr:street')] if v) + key = (normalize_city(props.get('county_postal_community')), + normalize_city(props.get('county_city')), + normalize_city(props.get('osm_city'))) + if key not in groups: + groups[key] = {'count': 0, 'addr': addr} + groups[key]['count'] += 1 + + mismatches = 0 + for (postal, city, osm), g in groups.items(): + fields = [('postal community', postal), ('city', city), ('OSM city', osm)] + parts = [] + row_mismatch = False + for field_label, value in fields: + if not value: + continue + agrees = (not official_city) or value.lower() == official_city.lower() + if not agrees: + row_mismatch = True + flag = '' if agrees else ' <-- MISMATCH' + parts.append(f"{field_label}={value}{flag}") + count_note = f" (x{g['count']})" if g['count'] > 1 else '' + if row_mismatch: + mismatches += g['count'] + print(f" {g['addr']}{count_note}: " + ', '.join(parts)) + + return {'checked': True, 'matched': True, 'mismatches': mismatches} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--reference', default=str(Path(__file__).parent / 'reference.yml')) + parser.add_argument('--data-dir', default='/data') + parser.add_argument('--radius', type=float, default=DEFAULT_RADIUS_METERS, + help=f'Match radius in meters for lat/lon entries (default {DEFAULT_RADIUS_METERS})') + args = parser.parse_args() + + ref_path = Path(args.reference) + if not ref_path.exists(): + print(f"Error: {ref_path} not found.", file=sys.stderr) + return 1 + + with open(ref_path) as f: + data = yaml.safe_load(f) or {} + checks = data.get('checks') or [] + + if not checks: + print(f"No entries in {ref_path} yet.") + print("Open https://www.thevillages.com/map/ in your own browser, look up an address,") + print("and add an entry to reference.yml by hand - see the comments at the top of that file.") + return 0 + + data_dir = Path(args.data_dir) + cache = {} + total_mismatches = 0 + total_unmatched = 0 + + for i, entry in enumerate(checks, 1): + county = entry.get('county') + if not county: + print(f"\n[{i}] Skipped - missing 'county' field.") + continue + if county not in cache: + cache[county] = ( + load_geojson(data_dir / county / 'county-addresses.geojson'), + load_geojson(data_dir / county / 'city-comparison.geojson'), + ) + county_features, comparison_features = cache[county] + result = check_entry(entry, i, county_features, comparison_features, args.radius) + if not result['matched']: + total_unmatched += 1 + total_mismatches += result['mismatches'] + + print(f"\n{'=' * 60}") + print(f"Checked {len(checks)} entries: {total_mismatches} address(es) with a " + f"disagreeing city field, {total_unmatched} entries with no match in our data.") + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/map-checker/reference.yml b/map-checker/reference.yml new file mode 100644 index 0000000..79cdce2 --- /dev/null +++ b/map-checker/reference.yml @@ -0,0 +1,32 @@ +# map-checker reference log +# +# This file is filled in BY HAND. Open https://www.thevillages.com/map/ in +# your own browser, look up a street/area, and record what the official map +# says here. Nothing in this subproject fetches, scrapes, or otherwise +# automates access to thevillages.com — their Terms of Use explicitly +# prohibit automated/bot access ("You must not use any robot, spider or +# other automatic device, process or means to access the Website for any +# purpose, including monitoring or copying any of the material on the +# Website"), so every entry here is a manual, human-recorded observation. +# +# Fields: +# street - street name as it appears in county/OSM data (required +# unless lat/lon given instead) +# housenumber - optional, narrows the match to one address +# lat / lon - optional alternative to street: a point you looked up +# on the official map (e.g. right-clicked or geocoded) +# county - which county's data to check against: sumter | lake +# official_village - the named Village shown on thevillages.com/map (e.g. +# "Fenney", "DeLuna"), if shown - informational only +# official_city - what city/community the official map (or other +# authoritative source you consulted) indicates this +# address belongs to +# recorded_by - your name/username +# recorded_date - YYYY-MM-DD +# notes - anything else worth remembering (screenshots taken, +# ambiguity noticed, source other than the map, etc.) +# +# Run `python check.py` after adding entries to compare them against our +# county/OSM data and flag disagreements. + +checks: [] diff --git a/web/server.py b/web/server.py index 9b2ef5d..b111a8b 100644 --- a/web/server.py +++ b/web/server.py @@ -9,6 +9,7 @@ import subprocess import os import re import secrets +import shlex import shutil import threading import json @@ -185,6 +186,117 @@ def map_viewer(): """Map viewer page""" return render_template('map.html') +# ── Reports ──────────────────────────────────────────────────────────────── + +@app.route('/reports') +def reports_page(): + """Reports index page""" + return render_template('reports.html', current_user=session.get('username')) + +@app.route('/reports/city-boundaries') +def city_boundaries_report(): + """City name agreement report: county vs OSM addr:city, on a map""" + counties = get_counties() + available = [] + pinned_cities = {} + for county in counties: + path = f'/data/{county["id"]}/city-comparison.geojson' + if os.path.isfile(path): + available.append(county['id']) + pinned_cities[county['id']] = county.get('pinned_cities') or [] + return render_template('city-boundaries-report.html', + counties=counties, + available_counties=available, + pinned_cities=pinned_cities, + current_user=session.get('username')) + +# ── map-checker: hand-traced boundary tool ────────────────────────────────── +# +# thevillages.com's Terms of Use prohibit automated access, so this only ever +# saves what a human draws by hand while looking at the official map +# themselves - nothing here fetches from thevillages.com. See map-checker/README.md. + +MAP_CHECKER_DIR = '/data/map-checker' +MAP_CHECKER_BOUNDARY_FILE = os.path.join(MAP_CHECKER_DIR, 'boundary.geojson') +MAP_CHECKER_DRAFT_FILE = os.path.join(MAP_CHECKER_DIR, 'draft.json') + +@app.route('/map-checker') +@require_auth +def map_checker_page(): + """Boundary tracer: hand-trace a polygon against a reference map, save as GeoJSON. + + Requires login so tracing work is never done (and lost) as an anonymous session.""" + return render_template('boundary-tracer.html', + counties=get_counties(), + current_user=session.get('username')) + +@app.route('/api/map-checker/boundary', methods=['GET']) +@require_auth +def get_map_checker_boundary(): + if not os.path.isfile(MAP_CHECKER_BOUNDARY_FILE): + return jsonify({'type': 'FeatureCollection', 'features': []}) + with open(MAP_CHECKER_BOUNDARY_FILE) as f: + return jsonify(json.load(f)) + +@app.route('/api/map-checker/boundary', methods=['POST']) +@require_auth +def save_map_checker_boundary(): + data = request.json or {} + if data.get('type') != 'FeatureCollection' or not isinstance(data.get('features'), list): + return jsonify({'error': 'Expected a GeoJSON FeatureCollection'}), 400 + for feat in data['features']: + geom_type = ((feat.get('geometry') or {}).get('type')) + if geom_type not in ('Polygon', 'MultiPolygon'): + return jsonify({'error': f'Unsupported geometry type: {geom_type}'}), 400 + os.makedirs(MAP_CHECKER_DIR, exist_ok=True) + tmp = MAP_CHECKER_BOUNDARY_FILE + '.tmp' + with open(tmp, 'w') as f: + json.dump(data, f) + os.replace(tmp, MAP_CHECKER_BOUNDARY_FILE) + return jsonify({'success': True, 'features': len(data['features'])}) + + +def _is_latlng_list(val): + if not isinstance(val, list): + return False + for pt in val: + if not (isinstance(pt, list) and len(pt) == 2 and + all(isinstance(c, (int, float)) for c in pt)): + return False + return True + + +@app.route('/api/map-checker/draft', methods=['GET']) +@require_auth +def get_map_checker_draft(): + if not os.path.isfile(MAP_CHECKER_DRAFT_FILE): + return jsonify({'polygons': [], 'inProgress': []}) + with open(MAP_CHECKER_DRAFT_FILE) as f: + return jsonify(json.load(f)) + +@app.route('/api/map-checker/draft', methods=['POST']) +@require_auth +def save_map_checker_draft(): + """Autosave endpoint hit after every vertex/edit so in-progress tracing + survives a crashed tab or dropped connection, even before the user + explicitly finishes a polygon or clicks Save to server.""" + data = request.json or {} + polygons = data.get('polygons') + in_progress = data.get('inProgress', []) + if not isinstance(polygons, list): + return jsonify({'error': 'Expected a polygons list'}), 400 + for p in polygons: + if not (isinstance(p, dict) and isinstance(p.get('name'), str) and _is_latlng_list(p.get('latlngs'))): + return jsonify({'error': 'Malformed polygon entry'}), 400 + if not _is_latlng_list(in_progress): + return jsonify({'error': 'Malformed inProgress point list'}), 400 + os.makedirs(MAP_CHECKER_DIR, exist_ok=True) + tmp = MAP_CHECKER_DRAFT_FILE + '.tmp' + with open(tmp, 'w') as f: + json.dump({'polygons': polygons, 'inProgress': in_progress}, f) + os.replace(tmp, MAP_CHECKER_DRAFT_FILE) + return jsonify({'success': True}) + def get_script_map(): """Build script map dynamically from county configuration.""" counties = get_counties() @@ -197,6 +309,7 @@ def get_script_map(): osm_roads = {}; osm_addrs = {}; osm_paths = {} conv_roads = {}; conv_addrs = {}; conv_paths = {} diff_roads = {}; diff_addrs = {}; diff_paths = {} + city_report = {} for c in counties: cid = c['id'] @@ -226,6 +339,7 @@ def get_script_map(): diff_roads[cid] = ['python', 'diff-highways.py', f'{base}/osm-roads.geojson', f'{base}/county-roads.geojson', '--output', f'{base}/diff-roads.geojson'] diff_addrs[cid] = ['python', 'compare-addresses.py', '--local-file', f'{base}/county-addresses.geojson', '--osm-file', f'{base}/osm-addresses.geojson', '--output-dir', base] + city_report[cid] = ['python', 'generate-city-report.py', '--existing-file', f'{base}/addresses-existing.geojson', '--output-dir', base] if pu: diff_paths[cid] = ['python', 'diff-highways.py', f'{base}/osm-paths.geojson', f'{base}/county-paths.geojson', '--output', f'{base}/diff-paths.geojson'] @@ -241,6 +355,19 @@ def get_script_map(): sm['diff-roads'] = diff_roads sm['diff-addresses'] = diff_addrs if diff_paths: sm['diff-paths'] = diff_paths + sm['generate-city-report'] = city_report + + # Point-in-polygon check against the traced/imported boundary at + # /data/map-checker/boundary.geojson (see /map-checker and map-checker/README.md). + # Not per-county like the scripts above - one boundary can span multiple + # counties, so this checks all configured counties in a single run. + county_args = ' '.join(f'--county {shlex.quote(c["id"])}' for c in counties) + if county_args: + sm['boundary-check'] = ( + f'python map-checker/check-boundary.py {county_args} ' + f'--boundary /data/map-checker/boundary.geojson ' + f'--output /data/map-checker/flagged.geojson' + ) return sm @@ -548,6 +675,7 @@ DEFAULT_COUNTIES = [ 'roads_url': 'https://www.arcgis.com/sharing/rest/content/items/9177e17c72d3433aa79630c7eda84add/data', 'addresses_url': 'https://www.arcgis.com/sharing/rest/content/items/c75c5aac13a648968c5596b0665be28b/data', 'paths_url': 'upload', + 'pinned_cities': ['The Villages'], }, { 'id': 'lake', @@ -557,6 +685,7 @@ DEFAULT_COUNTIES = [ 'roads_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Streets.zip', 'addresses_url': 'https://c.lakecountyfl.gov/ftp/GIS/GisDownloads/Shapefiles/Addresspoints.zip', 'paths_url': '', + 'pinned_cities': ['The Villages'], }, ] diff --git a/web/static/boundary-tracer.js b/web/static/boundary-tracer.js new file mode 100644 index 0000000..644a257 --- /dev/null +++ b/web/static/boundary-tracer.js @@ -0,0 +1,478 @@ +// Boundary Tracer: hand-click a polygon boundary while looking at a +// reference map in another tab, save it as GeoJSON for map-checker's +// point-in-polygon analysis. Tracing itself never fetches from any +// third-party site; the optional overlay layer is the one exception and +// only ever fetches a URL the user explicitly typed in. + +const POLYGON_COLORS = ['#2a78d6', '#eb6834', '#1baf7a', '#eda100', '#e87ba4', '#008300', '#4a3aa7', '#e34948']; + +let map; +let polygons = []; // { id, name, latlngs: [[lat,lng],...], layer: L.Polygon } +let nextPolygonId = 1; + +let drawingActive = false; +let drawingPoints = []; // [[lat,lng],...] +let drawingLayer = null; // preview L.Polyline/L.Polygon +let vertexMarkers = []; + +// Overlay layer: the user pastes either a standard {z}/{x}/{y} tile URL, or +// the base URL of an ArcGIS tiled MapServer. Both are fetched directly by +// the browser from whatever URL the user gave - never proxied server-side. +// +// A generic {z}/{x}/{y} template is assumed to already follow the standard +// Web Mercator slippy-tile grid and is rendered with a plain L.tileLayer. +// +// An ArcGIS MapServer may instead publish tiles on its own scheme (see that +// service's own `?f=json` -> tileInfo: origin, per-level resolution, tile +// size). Many such caches - including local government ones - use a plain +// geographic (WGS84, wkid 4326) grid rather than Web Mercator, which a +// standard tile client can't address with z/x/y math. We handle that case by +// computing each source tile's true lat/lng box from tileInfo and dropping +// it in as an L.imageOverlay, which Leaflet projects correctly onto the +// Mercator base map regardless of the source's own projection. +let overlayMode = null; // 'xyz' | 'arcgis-geo' +let overlayTileLayer = null; // 'xyz' mode: L.tileLayer +let overlayGroup = null; // 'arcgis-geo' mode: L.layerGroup of L.imageOverlay +let overlayArcGisMeta = null; // 'arcgis-geo' mode: { baseUrl, tileInfo, opacity } +const OVERLAY_STORAGE_KEY = 'boundary-tracer-overlay'; +const MAX_ARCGIS_OVERLAY_TILES = 80; + +function initMap() { + map = initBaseMap('map', [28.7, -81.7], 10); + map.on('click', onMapClick); +} + +function isArcGisMapServerUrl(url) { + return /\/MapServer\/?$/i.test(url); +} + +function addOverlay(url, opacity) { + removeOverlay(); + url = url.trim(); + if (isArcGisMapServerUrl(url)) { + addArcGisGeoOverlay(url, opacity); + } else { + overlayMode = 'xyz'; + overlayTileLayer = L.tileLayer(url, { opacity, maxZoom: 22, maxNativeZoom: 20 }).addTo(map); + document.getElementById('removeOverlayBtn').disabled = false; + document.getElementById('overlayOpacityRow').style.display = 'flex'; + } + try { localStorage.setItem(OVERLAY_STORAGE_KEY, JSON.stringify({ url, opacity })); } catch (e) {} +} + +function addArcGisGeoOverlay(baseUrl, opacity) { + const cleanUrl = baseUrl.replace(/\/$/, ''); + fetch(cleanUrl + '?f=json') + .then(r => r.json()) + .then(meta => { + const tileInfo = meta.tileInfo; + if (!tileInfo || !tileInfo.lods || !tileInfo.origin) { + showStatus('This ArcGIS service has no tileInfo - it may not be a cached/tiled MapServer.', 'error'); + return; + } + const wkid = (tileInfo.spatialReference || {}).wkid; + if (wkid !== 4326) { + showStatus(`Unsupported tile spatial reference (wkid ${wkid}). Only geographic (wkid 4326) ArcGIS tile schemes are supported.`, 'error'); + return; + } + overlayMode = 'arcgis-geo'; + overlayArcGisMeta = { baseUrl: cleanUrl, tileInfo, opacity }; + overlayGroup = L.layerGroup().addTo(map); + map.on('moveend', redrawArcGisOverlay); + document.getElementById('removeOverlayBtn').disabled = false; + document.getElementById('overlayOpacityRow').style.display = 'flex'; + redrawArcGisOverlay(); + showStatus('ArcGIS overlay loaded - it refreshes automatically as you pan/zoom.', 'success'); + }) + .catch(err => showStatus(`Failed to load ArcGIS service metadata: ${err.message}`, 'error')); +} + +function pickArcGisLevel(tileInfo) { + // Match the current Leaflet zoom to the tileInfo LOD with the closest + // resolution, using the standard Web Mercator meters-per-pixel formula + // converted to degrees. This is an approximation (good enough to pick + // the nearest of a handful of power-of-two levels) not an exact reprojection. + const zoom = map.getZoom(); + const metersPerPixel = 156543.03392 * Math.cos(map.getCenter().lat * Math.PI / 180) / Math.pow(2, zoom); + const degPerPixel = metersPerPixel / 111320; + let best = tileInfo.lods[0]; + let bestDiff = Infinity; + tileInfo.lods.forEach(lod => { + const diff = Math.abs(lod.resolution - degPerPixel); + if (diff < bestDiff) { bestDiff = diff; best = lod; } + }); + return best; +} + +function redrawArcGisOverlay() { + if (!overlayGroup || !overlayArcGisMeta) return; + overlayGroup.clearLayers(); + const { baseUrl, tileInfo, opacity } = overlayArcGisMeta; + const lod = pickArcGisLevel(tileInfo); + const tileSizeDeg = lod.resolution * tileInfo.cols; + const originX = tileInfo.origin.x, originY = tileInfo.origin.y; + const bounds = map.getBounds(); + + const colMin = Math.floor((bounds.getWest() - originX) / tileSizeDeg); + const colMax = Math.floor((bounds.getEast() - originX) / tileSizeDeg); + const rowMin = Math.floor((originY - bounds.getNorth()) / tileSizeDeg); + const rowMax = Math.floor((originY - bounds.getSouth()) / tileSizeDeg); + + const tileCount = (colMax - colMin + 1) * (rowMax - rowMin + 1); + if (tileCount > MAX_ARCGIS_OVERLAY_TILES) { + showStatus(`Overlay area too large at this zoom (${tileCount} tiles needed) - zoom in further.`, 'error'); + return; + } + + for (let row = rowMin; row <= rowMax; row++) { + for (let col = colMin; col <= colMax; col++) { + if (row < 0 || col < 0) continue; + const tileNorth = originY - row * tileSizeDeg; + const tileSouth = originY - (row + 1) * tileSizeDeg; + const tileWest = originX + col * tileSizeDeg; + const tileEast = originX + (col + 1) * tileSizeDeg; + const url = `${baseUrl}/tile/${lod.level}/${row}/${col}`; + L.imageOverlay(url, [[tileSouth, tileWest], [tileNorth, tileEast]], { opacity }).addTo(overlayGroup); + } + } +} + +function removeOverlay() { + if (overlayTileLayer) { map.removeLayer(overlayTileLayer); overlayTileLayer = null; } + if (overlayGroup) { map.removeLayer(overlayGroup); overlayGroup = null; } + map.off('moveend', redrawArcGisOverlay); + overlayArcGisMeta = null; + overlayMode = null; + document.getElementById('removeOverlayBtn').disabled = true; + document.getElementById('overlayOpacityRow').style.display = 'none'; + try { localStorage.removeItem(OVERLAY_STORAGE_KEY); } catch (e) {} +} + +function setOverlayOpacity(opacity) { + if (overlayMode === 'xyz' && overlayTileLayer) { + overlayTileLayer.setOpacity(opacity); + } else if (overlayMode === 'arcgis-geo' && overlayArcGisMeta) { + overlayArcGisMeta.opacity = opacity; + redrawArcGisOverlay(); + } + try { + const saved = JSON.parse(localStorage.getItem(OVERLAY_STORAGE_KEY) || 'null'); + if (saved) { saved.opacity = opacity; localStorage.setItem(OVERLAY_STORAGE_KEY, JSON.stringify(saved)); } + } catch (e) {} +} + +function restoreOverlayFromStorage() { + try { + const saved = JSON.parse(localStorage.getItem(OVERLAY_STORAGE_KEY) || 'null'); + if (saved && saved.url) { + document.getElementById('overlayUrlInput').value = saved.url; + document.getElementById('overlayOpacitySlider').value = Math.round((saved.opacity ?? 0.7) * 100); + addOverlay(saved.url, saved.opacity ?? 0.7); + } + } catch (e) {} +} + +function colorForIndex(i) { + return POLYGON_COLORS[i % POLYGON_COLORS.length]; +} + +function onMapClick(e) { + if (!drawingActive) return; + drawingPoints.push([e.latlng.lat, e.latlng.lng]); + redrawPreview(); + updateDrawButtons(); + saveDraft(); +} + +function redrawPreview() { + if (drawingLayer) { + map.removeLayer(drawingLayer); + drawingLayer = null; + } + vertexMarkers.forEach(m => map.removeLayer(m)); + vertexMarkers = []; + + if (drawingPoints.length === 0) return; + + if (drawingPoints.length >= 3) { + drawingLayer = L.polygon(drawingPoints, { + color: '#ffc107', weight: 2, fillOpacity: 0.15, dashArray: '5,5' + }).addTo(map); + } else { + drawingLayer = L.polyline(drawingPoints, { + color: '#ffc107', weight: 2, dashArray: '5,5' + }).addTo(map); + } + + drawingPoints.forEach(pt => { + const marker = L.circleMarker(pt, { + radius: 4, fillColor: '#ffc107', color: '#a67c00', weight: 1, fillOpacity: 1 + }).addTo(map); + vertexMarkers.push(marker); + }); +} + +function updateDrawButtons() { + document.getElementById('undoPointBtn').disabled = drawingPoints.length === 0; + document.getElementById('finishPolygonBtn').disabled = drawingPoints.length < 3; + const hint = document.getElementById('drawHint'); + if (drawingActive) { + hint.textContent = `${drawingPoints.length} point(s) placed. Click the map to add more, then "Finish polygon".`; + } +} + +function startDrawing() { + drawingActive = true; + drawingPoints = []; + redrawPreview(); + document.getElementById('newPolygonBtn').disabled = true; + document.getElementById('undoPointBtn').disabled = true; + document.getElementById('finishPolygonBtn').disabled = true; + document.getElementById('cancelPolygonBtn').disabled = false; + document.getElementById('drawHint').textContent = 'Click the map to place vertices.'; +} + +function undoPoint() { + if (drawingPoints.length === 0) return; + drawingPoints.pop(); + redrawPreview(); + updateDrawButtons(); + saveDraft(); +} + +function cancelDrawing() { + drawingActive = false; + drawingPoints = []; + if (drawingLayer) { map.removeLayer(drawingLayer); drawingLayer = null; } + vertexMarkers.forEach(m => map.removeLayer(m)); + vertexMarkers = []; + document.getElementById('newPolygonBtn').disabled = false; + document.getElementById('undoPointBtn').disabled = true; + document.getElementById('finishPolygonBtn').disabled = true; + document.getElementById('cancelPolygonBtn').disabled = true; + document.getElementById('drawHint').textContent = 'Click "New polygon" to start tracing.'; + saveDraft(); +} + +function finishDrawing() { + if (drawingPoints.length < 3) return; + const defaultName = polygons.length > 0 ? polygons[polygons.length - 1].name : 'The Villages'; + const name = window.prompt('Name for this polygon (e.g. "The Villages"):', defaultName); + if (name === null) return; // cancelled prompt - keep drawing state as-is + + const latlngs = drawingPoints.slice(); + if (drawingLayer) { map.removeLayer(drawingLayer); drawingLayer = null; } + vertexMarkers.forEach(m => map.removeLayer(m)); + vertexMarkers = []; + + addPolygon(name.trim() || 'Unnamed', latlngs); + + drawingActive = false; + drawingPoints = []; + document.getElementById('newPolygonBtn').disabled = false; + document.getElementById('undoPointBtn').disabled = true; + document.getElementById('finishPolygonBtn').disabled = true; + document.getElementById('cancelPolygonBtn').disabled = true; + document.getElementById('drawHint').textContent = 'Click "New polygon" to start tracing.'; + saveDraft(); +} + +function addPolygon(name, latlngs) { + const id = nextPolygonId++; + const color = colorForIndex(polygons.length); + const layer = L.polygon(latlngs, { color, weight: 2, fillOpacity: 0.2 }).addTo(map); + polygons.push({ id, name, latlngs, layer }); + renderPolygonList(); +} + +function deletePolygon(id) { + const idx = polygons.findIndex(p => p.id === id); + if (idx === -1) return; + map.removeLayer(polygons[idx].layer); + polygons.splice(idx, 1); + // Recolor remaining layers to keep the palette assignment stable-looking + polygons.forEach((p, i) => p.layer.setStyle({ color: colorForIndex(i) })); + renderPolygonList(); + saveDraft(); +} + +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, c => ({ + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' + }[c])); +} + +function renderPolygonList() { + const container = document.getElementById('polygonList'); + if (polygons.length === 0) { + container.innerHTML = '

None yet.

'; + return; + } + container.innerHTML = polygons.map((p, i) => ` +
+ + ${escapeHtml(p.name)} (${p.latlngs.length} pts) + +
+ `).join(''); +} +window.__deletePolygon = deletePolygon; + +function polygonsToGeoJson() { + return { + type: 'FeatureCollection', + features: polygons.map(p => ({ + type: 'Feature', + properties: { name: p.name }, + geometry: { + type: 'Polygon', + // GeoJSON is [lng, lat] and rings must close (first == last) + coordinates: [[...p.latlngs.map(([lat, lng]) => [lng, lat]), + [p.latlngs[0][1], p.latlngs[0][0]]]] + } + })) + }; +} + +function explodeGeometry(geom) { + if (geom.type === 'Polygon') return [geom.coordinates]; + if (geom.type === 'MultiPolygon') return geom.coordinates; + return []; +} + +function loadExisting() { + fetch('/api/map-checker/boundary') + .then(r => r.json()) + .then(geojson => { + (geojson.features || []).forEach(feat => { + const name = (feat.properties && feat.properties.name) || 'Unnamed'; + explodeGeometry(feat.geometry).forEach(rings => { + const outerRing = rings[0]; // ignore holes + const latlngs = outerRing.slice(0, -1).map(([lng, lat]) => [lat, lng]); // drop closing point + if (latlngs.length >= 3) addPolygon(name, latlngs); + }); + }); + if (polygons.length > 0) { + const bounds = L.latLngBounds(polygons.flatMap(p => p.latlngs)); + map.fitBounds(bounds, { padding: [50, 50] }); + } + }) + .catch(err => showStatus(`Failed to load existing boundary: ${err.message}`, 'error')); +} + +// Autosave: fired after every vertex click, undo, delete, and finish so an +// in-progress trace survives a crashed tab or dropped connection without +// needing an explicit "Save to server" click. +function saveDraft() { + fetch('/api/map-checker/draft', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + keepalive: true, + body: JSON.stringify({ + polygons: polygons.map(p => ({ name: p.name, latlngs: p.latlngs })), + inProgress: drawingPoints + }) + }).catch(err => showStatus(`Autosave failed: ${err.message}`, 'error')); +} + +function loadInitialState() { + fetch('/api/map-checker/draft') + .then(r => r.json()) + .then(draft => { + const hasDraft = draft && ( + (draft.polygons && draft.polygons.length > 0) || + (draft.inProgress && draft.inProgress.length > 0) + ); + if (!hasDraft) { + loadExisting(); + return; + } + (draft.polygons || []).forEach(p => { + if (p.latlngs && p.latlngs.length >= 3) addPolygon(p.name || 'Unnamed', p.latlngs); + }); + if (draft.inProgress && draft.inProgress.length > 0) { + drawingActive = true; + drawingPoints = draft.inProgress.slice(); + redrawPreview(); + document.getElementById('newPolygonBtn').disabled = true; + document.getElementById('cancelPolygonBtn').disabled = false; + updateDrawButtons(); + } + const allPts = polygons.flatMap(p => p.latlngs).concat(drawingPoints); + if (allPts.length > 0) { + map.fitBounds(L.latLngBounds(allPts), { padding: [50, 50] }); + } + showStatus('Restored autosaved work from your last session.', 'success'); + }) + .catch(err => showStatus(`Failed to load autosaved draft: ${err.message}`, 'error')); +} + +function showStatus(msg, type) { + const el = document.getElementById('status'); + el.textContent = msg; + el.className = `status ${type}`; +} + +function saveToServer() { + if (polygons.length === 0) { + showStatus('No polygons to save yet.', 'error'); + return; + } + fetch('/api/map-checker/boundary', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(polygonsToGeoJson()) + }) + .then(r => r.json()) + .then(data => { + if (data.error) { showStatus(`Error: ${data.error}`, 'error'); return; } + showStatus(`Saved ${data.features} polygon(s) to /data/map-checker/boundary.geojson`, 'success'); + }) + .catch(err => showStatus(`Error: ${err.message}`, 'error')); +} + +function downloadGeoJson() { + if (polygons.length === 0) { + showStatus('No polygons to download yet.', 'error'); + return; + } + const blob = new Blob([JSON.stringify(polygonsToGeoJson(), null, 2)], { type: 'application/geo+json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'boundary.geojson'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} + +document.addEventListener('DOMContentLoaded', function() { + initMap(); + initMobileControls(); + loadInitialState(); + restoreOverlayFromStorage(); + + document.getElementById('newPolygonBtn').addEventListener('click', startDrawing); + document.getElementById('undoPointBtn').addEventListener('click', undoPoint); + document.getElementById('finishPolygonBtn').addEventListener('click', finishDrawing); + document.getElementById('cancelPolygonBtn').addEventListener('click', cancelDrawing); + document.getElementById('saveBtn').addEventListener('click', saveToServer); + document.getElementById('downloadBtn').addEventListener('click', downloadGeoJson); + + document.getElementById('addOverlayBtn').addEventListener('click', () => { + const url = document.getElementById('overlayUrlInput').value.trim(); + if (!url) { showStatus('Enter a tile URL first.', 'error'); return; } + const hasXyzPlaceholders = /\{z\}/.test(url) && /\{x\}/.test(url) && /\{y\}/.test(url); + if (!hasXyzPlaceholders && !isArcGisMapServerUrl(url)) { + showStatus('URL should contain {z}, {x} and {y} placeholders, or be an ArcGIS .../MapServer base URL.', 'error'); + return; + } + const opacity = document.getElementById('overlayOpacitySlider').value / 100; + addOverlay(url, opacity); + }); + document.getElementById('removeOverlayBtn').addEventListener('click', removeOverlay); + document.getElementById('overlayOpacitySlider').addEventListener('input', (e) => { + setOverlayOpacity(e.target.value / 100); + }); +}); diff --git a/web/static/map-ui.css b/web/static/map-ui.css new file mode 100644 index 0000000..39e992e --- /dev/null +++ b/web/static/map-ui.css @@ -0,0 +1,207 @@ +/* Shared chrome for Leaflet map pages: full-screen map, the slide-out + controls panel, mobile hamburger toggle, and status banner. Page-specific + controls (layer lists, legends, buttons) are styled by each page. */ + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + height: 100vh; + display: flex; + flex-direction: column; +} + +#map { + flex: 1; + width: 100%; +} + +.controls { + position: absolute; + top: 10px; + right: 10px; + z-index: 1000; + background: white; + padding: 10px; + border-radius: 8px; + box-shadow: 0 2px 10px rgba(0,0,0,0.2); + min-width: 175px; + max-height: calc(100vh - 20px); + overflow-y: auto; +} + +.controls h3 { + margin: 0 0 6px 0; + font-size: 12px; + color: #333; +} + +.controls label { + display: flex; + align-items: center; + margin: 4px 0; + cursor: pointer; + font-size: 11px; +} + +.controls input[type="checkbox"] { + margin-right: 8px; +} + +.controls select { + width: 100%; + font-size: 11px; + padding: 3px; + border: 1px solid #ccc; + border-radius: 4px; +} + +.controls button { + width: 100%; + padding: 7px; + margin-top: 7px; + background: #007bff; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; + font-size: 12px; + font-weight: 500; +} + +.controls button:hover { + background: #0056b3; +} + +.controls button:disabled { + background: #ccc; + cursor: not-allowed; +} + +.status { + margin-top: 7px; + padding: 6px; + border-radius: 4px; + font-size: 11px; + text-align: center; +} + +.status.success { + background: #d4edda; + color: #155724; +} + +.status.error { + background: #f8d7da; + color: #721c24; +} + +.status.hidden { + display: none; +} + +.hamburger-button { + display: none; + position: absolute; + top: 10px; + right: 10px; + z-index: 1001; + background: white; + border: none; + border-radius: 4px; + padding: 10px; + cursor: pointer; + box-shadow: 0 2px 10px rgba(0,0,0,0.2); + width: 44px; + height: 44px; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 4px; +} + +.hamburger-button span { + display: block; + width: 24px; + height: 3px; + background: #333; + border-radius: 2px; + transition: all 0.3s ease; +} + +.hamburger-button.active span:nth-child(1) { + transform: rotate(45deg) translate(6px, 6px); +} + +.hamburger-button.active span:nth-child(2) { + opacity: 0; +} + +.hamburger-button.active span:nth-child(3) { + transform: rotate(-45deg) translate(6px, -6px); +} + +@media (max-width: 768px) { + .hamburger-button { + display: flex; + } + + .controls { + position: fixed; + top: 0; + right: -100%; + width: 80%; + max-width: 320px; + height: 100vh; + overflow-y: auto; + transition: right 0.3s ease; + border-radius: 0; + box-shadow: -2px 0 10px rgba(0,0,0,0.2); + padding-top: 50px; + } + + .controls.open { + right: 0; + } + + .controls h3 { + font-size: 13px; + } + + .controls label { + font-size: 12px; + } +} + +.overlay { + display: none; + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0,0,0,0.5); + z-index: 999; +} + +.overlay.active { + display: block; +} + +.btn-back { + display: block; + margin-bottom: 12px; + padding: 5px 10px; + background: #1a1a2e; + color: #ddd; + border-radius: 4px; + font-size: 12px; + font-weight: 600; + text-decoration: none; + text-align: center; +} +.btn-back:hover { background: #2d2d4e; color: #fff; } diff --git a/web/static/map-ui.js b/web/static/map-ui.js new file mode 100644 index 0000000..84e5b11 --- /dev/null +++ b/web/static/map-ui.js @@ -0,0 +1,41 @@ +// Shared chrome for Leaflet-based pages: base tile layer and the mobile +// hamburger/controls-drawer behavior used by both the editing map viewer +// and read-only reports. + +function initBaseMap(elementId, view, zoom) { + const map = L.map(elementId).setView(view || [28.7, -81.7], 12); + + L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { + attribution: '© OpenStreetMap contributors © CARTO', + subdomains: 'abcd', + maxZoom: 20 + }).addTo(map); + + if (zoom) map.setZoom(zoom); + return map; +} + +// Wires up the hamburger button / slide-out controls panel / overlay used +// on mobile. Expects #hamburgerButton, #controls, #overlay in the page. +function initMobileControls() { + const hamburgerButton = document.getElementById('hamburgerButton'); + const controls = document.getElementById('controls'); + const overlay = document.getElementById('overlay'); + if (!hamburgerButton || !controls || !overlay) return; + + function toggleControls() { + controls.classList.toggle('open'); + hamburgerButton.classList.toggle('active'); + overlay.classList.toggle('active'); + } + + function closeControls() { + controls.classList.remove('open'); + hamburgerButton.classList.remove('active'); + overlay.classList.remove('active'); + } + + window.closeMobileControls = closeControls; + hamburgerButton.addEventListener('click', toggleControls); + overlay.addEventListener('click', closeControls); +} diff --git a/web/static/map.js b/web/static/map.js index b388e14..99f9a39 100644 --- a/web/static/map.js +++ b/web/static/map.js @@ -23,7 +23,6 @@ let selectionBox = null; let multiSelectMode = false; // Mobile controls state -let mobileControlsOpen = false; let loadedCounty = ''; let loadedDataType = ''; @@ -43,13 +42,7 @@ let popupLatlng = null; // Initialize map function initMap() { - map = L.map('map').setView([28.7, -81.7], 12); - - L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { - attribution: '© OpenStreetMap contributors © CARTO', - subdomains: 'abcd', - maxZoom: 20 - }).addTo(map); + map = initBaseMap('map', [28.7, -81.7], 12); snapPreviewLayer = L.layerGroup().addTo(map); @@ -2103,30 +2096,8 @@ function toggleLayer(layerId, layer) { document.addEventListener('DOMContentLoaded', function() { initMap(); - // Mobile hamburger menu toggle - const hamburgerButton = document.getElementById('hamburgerButton'); - const controls = document.getElementById('controls'); - const overlay = document.getElementById('overlay'); - - function toggleControls() { - mobileControlsOpen = !mobileControlsOpen; - controls.classList.toggle('open'); - hamburgerButton.classList.toggle('active'); - overlay.classList.toggle('active'); - } - - function closeControls() { - mobileControlsOpen = false; - controls.classList.remove('open'); - hamburgerButton.classList.remove('active'); - overlay.classList.remove('active'); - } - - // Expose closeControls globally for other functions - window.closeMobileControls = closeControls; - - hamburgerButton.addEventListener('click', toggleControls); - overlay.addEventListener('click', closeControls); + // Mobile hamburger menu toggle (shared helper in map-ui.js) + initMobileControls(); // Layer toggles document.getElementById('osmToggle').addEventListener('change', function() { diff --git a/web/static/report-map.js b/web/static/report-map.js new file mode 100644 index 0000000..0ad691b --- /dev/null +++ b/web/static/report-map.js @@ -0,0 +1,597 @@ +// City Boundaries report. Two independent views over the same data: +// - Agreement Status: county vs OSM addr:city agreement, color-coded. +// - City Names: what one source (county or OSM) thinks each address's city +// is, color-coded by identity - a visual read of what that source's +// "boundaries" look like. No agreement analysis in this view. + +const STATUS_CONFIG = { + match: { color: '#28a745', label: 'Match' }, + mismatch: { color: '#dc3545', label: 'Mismatch' }, + county_only: { color: '#fd7e14', label: 'Missing in OSM' }, + osm_only: { color: '#17a2b8', label: 'Missing in county' }, + both_missing: { color: '#6c757d', label: 'Both missing' }, +}; + +// Categorical identity palette (dataviz skill reference instance, fixed hue +// order). Only city names among the 8 most common in this dataset get their +// own hue - the rest fold into "Other" per the skill's rule for identity +// channels with more categories than a palette can keep pairwise-distinct. +// Every marker is still identifiable exactly via its popup and the legend's +// text labels, so color is never the only channel carrying identity. +const CITY_PALETTE = ['#2a78d6', '#eb6834', '#1baf7a', '#eda100', '#e87ba4', '#008300', '#4a3aa7', '#e34948']; +const CITY_OTHER_COLOR = '#898781'; +const CITY_OTHER_LABEL = 'Other'; +const CITY_UNKNOWN_COLOR = '#c3c2b7'; +const CITY_UNKNOWN_LABEL = '(no city)'; +const CITY_UNKNOWN_KEY = '__unknown__'; +const CITY_OTHER_KEY = '__other__'; + +// Boundary Check view: point-in-polygon results from map-checker/check-boundary.py +// against the traced/imported boundary at /data/map-checker/boundary.geojson. +// Independent of the county-vs-OSM tag comparison above - this is a separate +// data source (an explicit reference boundary) checked with actual geometry +// rather than by comparing text fields. +const FLAG_CONFIG = { + inside_but_not_labeled: { color: '#dc3545', label: "Inside boundary, not labeled expected city" }, + outside_but_labeled: { color: '#fd7e14', label: "Outside boundary, labeled expected city" }, +}; + +let boundaryOutlineLayer = null; // L.geoJSON of the traced boundary polygon(s) +let boundaryFlagLayers = {}; // flag -> L.layerGroup +let boundaryLoaded = false; // avoid re-fetching every time the tab is switched +let boundaryJobId = null; +let boundaryLogInterval = null; + +let map; +let statusLayers = {}; // status -> L.layerGroup +let cityLayers = {}; // city name (or CITY_OTHER_KEY/CITY_UNKNOWN_KEY) -> L.layerGroup +let currentGeojson = null; +let currentView = 'agreement'; // 'agreement' | 'city-names' | 'boundary-check' +let compareField = 'postal_community'; // 'postal_community' | 'city' - which county attribute to compare against OSM +let citySource = 'county'; // 'county' | 'osm' - which side to color by, in the City Names view +let cityColorMap = {}; // canonical (lowercased) name -> hex, stable across county/osm toggle +let cityLabelMap = {}; // canonical name -> display label (most common raw casing) +let currentCountyId = null; + +// The county-side property name for the currently selected Compare field. +function countyFieldName() { + return compareField === 'city' ? 'county_city' : 'county_postal_community'; +} + +// The status property name (precomputed server-side) for the current field. +function statusFieldName() { + return compareField === 'city' ? 'status_city' : 'status_postal_community'; +} + +// The property actually driving color in the City Names view right now. +function activeCityField() { + return citySource === 'osm' ? 'osm_city' : countyFieldName(); +} + +function initMap() { + map = initBaseMap('map', [28.7, -81.7], 11); + Object.keys(STATUS_CONFIG).forEach(status => { + statusLayers[status] = L.layerGroup(); + }); + Object.keys(FLAG_CONFIG).forEach(flag => { + boundaryFlagLayers[flag] = L.layerGroup(); + }); +} + +function markerStyle(color) { + return { + radius: 6, + fillColor: color, + color: '#fff', + weight: 1, + opacity: 1, + fillOpacity: 0.85, + }; +} + +function escapeHtml(s) { + if (s === null || s === undefined) return ''; + return String(s).replace(/[&<>"']/g, c => ({ + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' + }[c])); +} + +function popupHtml(feature) { + const p = feature.properties || {}; + const activeStatus = p[statusFieldName()]; + const cfg = STATUS_CONFIG[activeStatus] || {}; + const compareLabel = compareField === 'city' ? 'city' : 'postal community'; + const addrParts = [p['addr:housenumber'], p['addr:street']].filter(Boolean).join(' '); + const unit = p['addr:unit'] ? ` Unit ${escapeHtml(p['addr:unit'])}` : ''; + + let html = '
'; + html += `
${escapeHtml(addrParts) || '(no address)'}${unit}
`; + html += `
County postal community: ${escapeHtml(p.county_postal_community) || 'none'}
`; + html += `
County city: ${escapeHtml(p.county_city) || 'none'}
`; + html += `
OSM city: ${escapeHtml(p.osm_city) || 'none'}
`; + html += `
Agreement (vs ${compareLabel}): ${escapeHtml(cfg.label || activeStatus)}
`; + if (p.distance_meters !== undefined && p.distance_meters !== null) { + html += `
Match distance: ${Number(p.distance_meters).toFixed(1)} m
`; + } + html += '
'; + return html; +} + +// ---- Agreement Status view ---- + +function updateStatusCounts(counts) { + Object.keys(STATUS_CONFIG).forEach(status => { + const el = document.getElementById(`count-${status}`); + if (el) el.textContent = counts[status] || 0; + }); +} + +function renderStatusLayers(geojson) { + Object.values(statusLayers).forEach(layer => layer.clearLayers()); + + const field = statusFieldName(); + const counts = {}; + geojson.features.forEach(feature => { + const status = feature.properties && feature.properties[field]; + const cfg = STATUS_CONFIG[status] || STATUS_CONFIG.both_missing; + const layer = statusLayers[status] || statusLayers.both_missing; + const coords = feature.geometry.coordinates; + const marker = L.circleMarker(L.latLng(coords[1], coords[0]), markerStyle(cfg.color)); + marker.bindPopup(popupHtml(feature), { maxWidth: 280 }); + layer.addLayer(marker); + counts[status] = (counts[status] || 0) + 1; + }); + + updateStatusCounts(counts); +} + +function applyStatusLayerVisibility() { + document.querySelectorAll('#legend .legend-item').forEach(item => { + const status = item.dataset.status; + const checked = item.querySelector('input[type="checkbox"]').checked; + const layer = statusLayers[status]; + if (!layer) return; + if (checked) layer.addTo(map); + else if (map.hasLayer(layer)) map.removeLayer(layer); + }); +} + +// ---- City Names view ---- + +function cityKey(name) { + const trimmed = name && String(name).trim(); + return trimmed ? trimmed : null; +} + +// Case-insensitive grouping key so source data noise like "Clermont" vs +// "CLermont" doesn't fragment into separate colors/legend rows. +function canonicalKey(rawName) { + return rawName ? rawName.toLowerCase() : null; +} + +// Colors are assigned once per dataset load, ranked by combined frequency +// across BOTH the active county field (postal community or city, per the +// Compare dropdown) and osm_city. This keeps a given city's color fixed +// regardless of which side (county/OSM) is currently selected or which +// layers are toggled - identity never gets repainted out from under the +// user. Colors ARE re-ranked when the Compare field itself changes, since +// that's genuinely a different attribute. +// +// pinnedNames (raw strings, e.g. from counties.yml's pinned_cities) always +// claim a palette slot ahead of frequency ranking, as long as they actually +// appear in the data - so a place like "The Villages" that a county considers +// important never gets folded into "Other" just because it's a numeric +// minority in this particular dataset. +function buildCityColorMap(geojson, pinnedNames, countyField) { + const counts = new Map(); // canonical key -> total count + const labelCounts = new Map(); // canonical key -> Map(raw label -> count) + + geojson.features.forEach(feature => { + const p = feature.properties || {}; + [cityKey(p[countyField]), cityKey(p.osm_city)].forEach(raw => { + if (!raw) return; + const key = canonicalKey(raw); + counts.set(key, (counts.get(key) || 0) + 1); + if (!labelCounts.has(key)) labelCounts.set(key, new Map()); + const lm = labelCounts.get(key); + lm.set(raw, (lm.get(raw) || 0) + 1); + }); + }); + + const rankedByFrequency = [...counts.keys()].sort((a, b) => counts.get(b) - counts.get(a)); + + const pinnedKeys = (pinnedNames || []) + .map(canonicalKey) + .filter(key => counts.has(key)); // only pin names that actually occur here + const pinnedSet = new Set(pinnedKeys); + + // Pinned names first (in the order given), then the rest by frequency. + const ranked = [...pinnedKeys, ...rankedByFrequency.filter(key => !pinnedSet.has(key))]; + + cityColorMap = {}; + cityLabelMap = {}; + ranked.forEach((key, i) => { + cityColorMap[key] = i < CITY_PALETTE.length ? CITY_PALETTE[i] : CITY_OTHER_COLOR; + // Display label: whichever raw casing was most common for this key. + let bestLabel = key, bestCount = -1; + labelCounts.get(key).forEach((c, label) => { + if (c > bestCount) { bestCount = c; bestLabel = label; } + }); + cityLabelMap[key] = bestLabel; + }); +} + +function colorForCity(canonKey) { + if (canonKey === null) return CITY_UNKNOWN_COLOR; + return cityColorMap[canonKey] || CITY_OTHER_COLOR; +} + +function renderCityLayers(geojson, field) { + Object.values(cityLayers).forEach(layer => { layer.clearLayers(); if (map.hasLayer(layer)) map.removeLayer(layer); }); + cityLayers = {}; + + // Per-field counts for the legend (independent of the shared color ranking). + const counts = new Map(); + + geojson.features.forEach(feature => { + const raw = cityKey((feature.properties || {})[field]); + const canon = canonicalKey(raw); + const color = colorForCity(canon); + // Named (palette) cities each get their own bucket; everything folded + // to CITY_OTHER_COLOR shares a single "Other" bucket/legend row. + let bucketKey; + if (canon === null) bucketKey = CITY_UNKNOWN_KEY; + else if (color === CITY_OTHER_COLOR) bucketKey = CITY_OTHER_KEY; + else bucketKey = canon; + + if (!cityLayers[bucketKey]) cityLayers[bucketKey] = L.layerGroup(); + + const coords = feature.geometry.coordinates; + const marker = L.circleMarker(L.latLng(coords[1], coords[0]), markerStyle(color)); + marker.bindPopup(popupHtml(feature), { maxWidth: 280 }); + cityLayers[bucketKey].addLayer(marker); + + counts.set(bucketKey, (counts.get(bucketKey) || 0) + 1); + }); + + renderCityLegend(counts); + applyCityLayerVisibility(); +} + +function renderCityLegend(counts) { + const container = document.getElementById('cityLegend'); + container.innerHTML = ''; + + // Named cities first, ranked by their assigned palette slot (fixed order), + // then Other, then Unknown - each only if present in this field's data. + const namedEntries = Object.keys(cityColorMap) + .filter(key => counts.has(key) && cityColorMap[key] !== CITY_OTHER_COLOR) + .sort((a, b) => CITY_PALETTE.indexOf(cityColorMap[a]) - CITY_PALETTE.indexOf(cityColorMap[b])); + + const rows = namedEntries.map(key => ({ key, label: cityLabelMap[key], color: cityColorMap[key] })); + if (counts.has(CITY_OTHER_KEY)) rows.push({ key: CITY_OTHER_KEY, label: CITY_OTHER_LABEL, color: CITY_OTHER_COLOR }); + if (counts.has(CITY_UNKNOWN_KEY)) rows.push({ key: CITY_UNKNOWN_KEY, label: CITY_UNKNOWN_LABEL, color: CITY_UNKNOWN_COLOR }); + + rows.forEach(({ key, label, color }) => { + const item = document.createElement('label'); + item.className = 'legend-item'; + item.dataset.cityKey = key; + item.innerHTML = ` + + + ${escapeHtml(label)} + ${counts.get(key) || 0} + `; + item.querySelector('input').addEventListener('change', function() { + const layer = cityLayers[key]; + if (!layer) return; + if (this.checked) layer.addTo(map); + else if (map.hasLayer(layer)) map.removeLayer(layer); + }); + container.appendChild(item); + }); +} + +function applyCityLayerVisibility() { + document.querySelectorAll('#cityLegend .legend-item').forEach(item => { + const key = item.dataset.cityKey; + const checked = item.querySelector('input[type="checkbox"]').checked; + const layer = cityLayers[key]; + if (!layer) return; + if (checked) layer.addTo(map); + else if (map.hasLayer(layer)) map.removeLayer(layer); + }); +} + +// ---- Boundary Check view ---- + +function boundaryPopupHtml(feature) { + const p = feature.properties || {}; + const cfg = FLAG_CONFIG[p.flag] || {}; + const addrParts = [p['addr:housenumber'], p['addr:street']].filter(Boolean).join(' '); + const unit = p['addr:unit'] ? ` Unit ${escapeHtml(p['addr:unit'])}` : ''; + + let html = '
'; + html += `
${escapeHtml(addrParts) || '(no address)'}${unit}
`; + html += `
County: ${escapeHtml(p.county) || ''}
`; + html += `
County postal community: ${escapeHtml(p.county_postal_community) || 'none'}
`; + html += `
County city: ${escapeHtml(p.county_city) || 'none'}
`; + html += `
OSM city: ${escapeHtml(p.osm_city) || 'none'}
`; + html += `
Flag: ${escapeHtml(cfg.label || p.flag)}
`; + html += '
'; + return html; +} + +function renderBoundaryOutline(geojson) { + if (boundaryOutlineLayer) { map.removeLayer(boundaryOutlineLayer); boundaryOutlineLayer = null; } + boundaryOutlineLayer = L.geoJSON(geojson, { + style: { color: '#6f42c1', weight: 2, fillOpacity: 0.05 } + }).bindPopup(layer => (layer.feature.properties || {}).name || 'Boundary'); +} + +function renderBoundaryFlags(geojson) { + Object.values(boundaryFlagLayers).forEach(layer => layer.clearLayers()); + + const counts = {}; + (geojson.features || []).forEach(feature => { + const flag = feature.properties && feature.properties.flag; + const layer = boundaryFlagLayers[flag]; + if (!layer) return; // unknown flag value - skip rather than guess a bucket + const cfg = FLAG_CONFIG[flag]; + const coords = feature.geometry.coordinates; + const marker = L.circleMarker(L.latLng(coords[1], coords[0]), markerStyle(cfg.color)); + marker.bindPopup(boundaryPopupHtml(feature), { maxWidth: 280 }); + layer.addLayer(marker); + counts[flag] = (counts[flag] || 0) + 1; + }); + + Object.keys(FLAG_CONFIG).forEach(flag => { + const el = document.getElementById(`count-${flag}`); + if (el) el.textContent = counts[flag] || 0; + }); +} + +function applyBoundaryLayerVisibility() { + document.querySelectorAll('#boundaryLegend .legend-item').forEach(item => { + const flag = item.dataset.flag; + const checked = item.querySelector('input[type="checkbox"]').checked; + const layer = flag === '__outline__' ? boundaryOutlineLayer : boundaryFlagLayers[flag]; + if (!layer) return; + if (checked) layer.addTo(map); + else if (map.hasLayer(layer)) map.removeLayer(layer); + }); +} + +function loadBoundarySection(force) { + if (boundaryLoaded && !force) { + applyBoundaryLayerVisibility(); + return; + } + + fetch('/data/map-checker/boundary.geojson') + .then(r => r.ok ? r.json() : null) + .then(geojson => { + if (boundaryOutlineLayer) { map.removeLayer(boundaryOutlineLayer); boundaryOutlineLayer = null; } + if (geojson && (geojson.features || []).length) renderBoundaryOutline(geojson); + if (currentView === 'boundary-check') applyBoundaryLayerVisibility(); + }) + .catch(() => {}); + + fetch('/data/map-checker/flagged.geojson') + .then(resp => { + if (!resp.ok) throw new Error('not generated yet'); + const lastMod = resp.headers.get('Last-Modified'); + document.getElementById('boundaryMeta').textContent = lastMod ? `Last run: ${new Date(lastMod).toLocaleString()}` : ''; + return resp.json(); + }) + .then(geojson => { + document.getElementById('boundaryNoData').style.display = 'none'; + renderBoundaryFlags(geojson); + boundaryLoaded = true; + if (currentView === 'boundary-check') applyBoundaryLayerVisibility(); + }) + .catch(() => { + document.getElementById('boundaryNoData').style.display = ''; + document.getElementById('boundaryMeta').textContent = ''; + Object.keys(FLAG_CONFIG).forEach(flag => { + const el = document.getElementById(`count-${flag}`); + if (el) el.textContent = '–'; + }); + if (currentView === 'boundary-check' && boundaryOutlineLayer) applyBoundaryLayerVisibility(); + }); +} + +function requireLoginForBoundaryCheck() { + showStatus('Not authenticated – please log in.', 'error'); + setTimeout(() => { window.location.href = '/login?next=' + encodeURIComponent(window.location.pathname); }, 1200); +} + +function runBoundaryCheck() { + if (!IS_AUTHENTICATED) return requireLoginForBoundaryCheck(); + + const btn = document.getElementById('runBoundaryCheckBtn'); + const logEl = document.getElementById('boundaryLog'); + btn.disabled = true; + logEl.style.display = 'block'; + logEl.textContent = ''; + document.getElementById('boundaryMeta').textContent = 'Running…'; + + fetch('/api/run-script', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ script: 'boundary-check' }) + }) + .then(r => r.json()) + .then(data => { + if (data.error) { + logEl.textContent = `Error: ${data.error}`; + btn.disabled = false; + return; + } + boundaryJobId = data.job_id; + if (boundaryLogInterval) clearInterval(boundaryLogInterval); + boundaryLogInterval = setInterval(checkBoundaryJobStatus, 1000); + }) + .catch(err => { + logEl.textContent = `Error: ${err.message}`; + btn.disabled = false; + }); +} + +function checkBoundaryJobStatus() { + if (!boundaryJobId) return; + fetch(`/api/job-status/${boundaryJobId}`) + .then(r => r.json()) + .then(data => { + const logEl = document.getElementById('boundaryLog'); + logEl.textContent = data.logs.join(''); + logEl.scrollTop = logEl.scrollHeight; + if (!data.running) { + clearInterval(boundaryLogInterval); + boundaryJobId = null; + document.getElementById('runBoundaryCheckBtn').disabled = false; + loadBoundarySection(true); + } + }) + .catch(err => console.error('Boundary check status error:', err)); +} + +// ---- View switching ---- + +function setView(view) { + currentView = view; + + document.querySelectorAll('.view-btn').forEach(btn => { + btn.classList.toggle('active', btn.dataset.view === view); + }); + document.getElementById('agreementSection').style.display = view === 'agreement' ? '' : 'none'; + document.getElementById('cityNameSection').style.display = view === 'city-names' ? '' : 'none'; + document.getElementById('boundarySection').style.display = view === 'boundary-check' ? '' : 'none'; + document.getElementById('countySelect').closest('.file-input-group').style.display = view === 'boundary-check' ? 'none' : ''; + document.getElementById('fieldSelect').closest('.file-input-group').style.display = view === 'boundary-check' ? 'none' : ''; + + Object.values(statusLayers).forEach(layer => { if (map.hasLayer(layer)) map.removeLayer(layer); }); + Object.values(cityLayers).forEach(layer => { if (map.hasLayer(layer)) map.removeLayer(layer); }); + Object.values(boundaryFlagLayers).forEach(layer => { if (map.hasLayer(layer)) map.removeLayer(layer); }); + if (boundaryOutlineLayer && map.hasLayer(boundaryOutlineLayer)) map.removeLayer(boundaryOutlineLayer); + + if (view === 'boundary-check') { + loadBoundarySection(); + return; + } + if (!currentGeojson) return; + if (view === 'agreement') { + applyStatusLayerVisibility(); + } else { + renderCityLayers(currentGeojson, activeCityField()); + } +} + +// ---- Data loading ---- + +function renderGeojson(geojson) { + currentGeojson = geojson; + + renderStatusLayers(geojson); + const pinned = (currentCountyId && PINNED_CITIES[currentCountyId]) || []; + buildCityColorMap(geojson, pinned, countyFieldName()); + + const bounds = L.latLngBounds([]); + geojson.features.forEach(feature => { + const coords = feature.geometry.coordinates; + bounds.extend(L.latLng(coords[1], coords[0])); + }); + if (bounds.isValid()) map.fitBounds(bounds, { padding: [50, 50] }); + + setView(currentView); +} + +function showStatus(msg, type) { + const el = document.getElementById('status'); + el.textContent = msg; + el.className = `status ${type}`; +} + +function hideStatus() { + const el = document.getElementById('status'); + el.className = 'status hidden'; +} + +function loadCounty(countyId) { + if (!countyId) return; + currentCountyId = countyId; + hideStatus(); + showStatus(`Loading ${countyId}...`, 'success'); + + fetch(`/data/${countyId}/city-comparison.geojson`) + .then(resp => { + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + return resp.json(); + }) + .then(geojson => { + renderGeojson(geojson); + hideStatus(); + }) + .catch(err => { + showStatus(`No city comparison data for ${countyId} yet. Run "Diff vs OSM" then "Generate City Boundaries Report" first. (${err.message})`, 'error'); + }); +} + +document.addEventListener('DOMContentLoaded', function() { + initMap(); + initMobileControls(); + + document.querySelectorAll('#legend .legend-item input[type="checkbox"]').forEach(cb => { + cb.addEventListener('change', function() { + const item = this.closest('.legend-item'); + const layer = statusLayers[item.dataset.status]; + if (!layer) return; + if (this.checked) layer.addTo(map); + else if (map.hasLayer(layer)) map.removeLayer(layer); + }); + }); + + document.querySelectorAll('.view-btn').forEach(btn => { + btn.addEventListener('click', () => setView(btn.dataset.view)); + }); + + document.querySelectorAll('#boundaryLegend .legend-item input[type="checkbox"]').forEach(cb => { + cb.addEventListener('change', applyBoundaryLayerVisibility); + }); + document.getElementById('runBoundaryCheckBtn').addEventListener('click', runBoundaryCheck); + + document.getElementById('citySourceSelect').addEventListener('change', function() { + citySource = this.value; + if (currentGeojson && currentView === 'city-names') { + renderCityLayers(currentGeojson, activeCityField()); + } + }); + + document.getElementById('fieldSelect').addEventListener('change', function() { + compareField = this.value; + if (!currentGeojson) return; + + const pinned = (currentCountyId && PINNED_CITIES[currentCountyId]) || []; + buildCityColorMap(currentGeojson, pinned, countyFieldName()); + + if (currentView === 'agreement') { + renderStatusLayers(currentGeojson); + applyStatusLayerVisibility(); + } else { + renderCityLayers(currentGeojson, activeCityField()); + } + }); + + const countySelect = document.getElementById('countySelect'); + countySelect.addEventListener('change', function() { + loadCounty(this.value); + }); + + const firstAvailable = AVAILABLE_COUNTIES && AVAILABLE_COUNTIES.length ? AVAILABLE_COUNTIES[0] : null; + if (firstAvailable) { + countySelect.value = firstAvailable; + loadCounty(firstAvailable); + } else { + showStatus('No counties have city comparison data yet. Run "Diff vs OSM" then "Generate City Boundaries Report" first.', 'error'); + } +}); diff --git a/web/templates/boundary-tracer.html b/web/templates/boundary-tracer.html new file mode 100644 index 0000000..07e8d19 --- /dev/null +++ b/web/templates/boundary-tracer.html @@ -0,0 +1,173 @@ + + + + + + Boundary Tracer + + + + + +
+ + + +
+ +
+ ← Back +

Boundary Tracer

+

+ Open a reference map (e.g. thevillages.com/map) in another tab, then + hand-trace its boundary here by clicking vertices. Tracing itself + never fetches from any third-party site — you're just drawing on + our own map based on what you see elsewhere. The optional overlay + layer below is the exception: it fetches tiles live from whatever + URL you give it. +

+ +

Overlay layer (optional)

+

+ This fetches tiles directly from the URL you paste in. Check that + source's Terms of Use permit automated/robotic access first — + thevillages.com's ToS, for example, explicitly prohibit it, so + don't point this at their tile servers. +

+

+ Accepts a standard {z}/{x}/{y} tile URL, or the base + URL of an ArcGIS tiled MapServer (e.g. .../MapServer) + — its own tiling scheme (including a plain-geographic/WGS84 grid) + is read automatically from that service's metadata. +

+ +
+ + +
+ + + + + + +

Click "New polygon" to start tracing.

+ +

Polygons

+

None yet.

+ + + + + +
+ + + + + + diff --git a/web/templates/city-boundaries-report.html b/web/templates/city-boundaries-report.html new file mode 100644 index 0000000..c7be05a --- /dev/null +++ b/web/templates/city-boundaries-report.html @@ -0,0 +1,243 @@ + + + + + + City Boundaries Report + + + + + +
+ + + +
+ +
+ ← Back to Reports +

City Boundaries

+

+ County source data vs matched OSM addresses. +

+ +
+ + +
+ +
+ + + These two often differ — a parcel can be legally inside one city but addressed to another for mail. +
+ +
+ + + +
+ +
+

Legend

+
+ + + + + +
+

"Match" is unchecked by default (usually the majority of points) — check it to see agreeing addresses too.

+
+ + + + + + +
+ + + + + + + diff --git a/web/templates/index.html b/web/templates/index.html index e0db255..a285aa7 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -361,6 +361,8 @@ Counties Street Exceptions Exclusions + Reports + Boundary Tracer Open Map Viewer
{% if current_user %} @@ -488,6 +490,13 @@ +
+
5
+
Generate City Boundaries Report
+
+ +
+
diff --git a/web/templates/map.html b/web/templates/map.html index 60b74ce..3e968ec 100644 --- a/web/templates/map.html +++ b/web/templates/map.html @@ -5,53 +5,8 @@ GeoJSON Map Viewer + @@ -380,6 +183,7 @@ + diff --git a/web/templates/reports.html b/web/templates/reports.html new file mode 100644 index 0000000..1ddb840 --- /dev/null +++ b/web/templates/reports.html @@ -0,0 +1,115 @@ + + + + + + Reports — OSM Import Tools + + + + +
+

OSM Import Tools

+
+ ← Back +
+ {% if current_user %} + {{ current_user }} + {% endif %} +
+ +
+

Reports

+ +

City Boundaries

+

Checks addr:city agreement between the county source data and OpenStreetMap for + matched addresses, plotted on a map and color-coded by agreement status.

+ +
+ + +