From 33224d9fd1e3623b29e9b9dfeb3475a254695217 Mon Sep 17 00:00:00 2001
From: zyphlar <zyphlar@gmail.com>
Date: Fri, 3 Apr 2026 23:07:33 -0700
Subject: [PATCH] get address efficiency by getting only centroids

---
 download-overpass.py | 99 +++++++++++++++++++++++++++-----------------
 1 file changed, 62 insertions(+), 37 deletions(-)

diff --git a/download-overpass.py b/download-overpass.py
index 2e34bf0..11d65aa 100644
--- a/download-overpass.py
+++ b/download-overpass.py
@@ -7,8 +7,6 @@ Usage:
     python download-overpass.py --type addresses "Lake County" "Florida" output/addresses.geojson
     python download-overpass.py --type multimodal "Sumter County" "Florida" output/paths.geojson
 
-TODO:
-- Don't just download roads. Probably ignore relations also.
 """
 
 import argparse
@@ -49,9 +47,19 @@ def build_overpass_query(county_name, state_name, data_type="highways"):
     """Build Overpass API query for specified data type in a county."""
     area_id = get_county_area_id(county_name, state_name)
 
-    base_query = f"""[out:json][timeout:60];
+    # Addresses: include nwr so building ways and relations are captured too,
+    # but use "out center;" instead of "out geom;" so ways/relations return a
+    # single centroid rather than their full coordinate list — much smaller payload.
+    if data_type == "addresses":
+        query = f"""[out:json][timeout:180];
+area(id:{area_id})->.searchArea;
+nwr["addr:housenumber"](area.searchArea);
+out center;"""
+        return query
+
+    base_query = f"""[out:json][timeout:120];
 area(id:{area_id})->.searchArea;"""
-    
+
     if data_type == "highways":
         selector = '('
         selector += 'way["highway"="motorway"](area.searchArea);'
@@ -65,68 +73,85 @@ area(id:{area_id})->.searchArea;"""
         selector += 'way["highway"="service"](area.searchArea);'
         selector += 'way["highway"="track"](area.searchArea);'
         selector += ');'
-    elif data_type == "addresses":
-        selector = 'nwr["addr:housenumber"](area.searchArea);'
     elif data_type == "multimodal":
         selector = '(way["highway"="path"](area.searchArea);way["highway"="cycleway"](area.searchArea););'
     else:
         raise ValueError(f"Unknown data type: {data_type}")
-    
+
     query = base_query + selector + "out geom;"
     return query
 
 
 def query_overpass(query):
-    """Send query to Overpass API and return JSON response."""
+    """Send query to Overpass API and return JSON response, with retries."""
     url = "https://overpass-api.de/api/interpreter"
     data = urllib.parse.urlencode({"data": query}).encode("utf-8")
-    
-    
-    try:
-        with urllib.request.urlopen(url, data=data) as response:
-            return json.loads(response.read().decode("utf-8"))
-    except urllib.error.HTTPError as e:
-        print(f"HTTP Error {e.code}: {e.reason}", file=sys.stderr)
+
+    max_attempts = 3
+    for attempt in range(1, max_attempts + 1):
         try:
-            error_body = e.read().decode("utf-8")
+            with urllib.request.urlopen(url, data=data, timeout=300) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as e:
+            try:
+                error_body = e.read().decode("utf-8")
+            except Exception:
+                error_body = "(unreadable)"
+            print(f"HTTP Error {e.code} on attempt {attempt}: {e.reason}", file=sys.stderr)
             print(f"Error response body: {error_body}", file=sys.stderr)
-        except:
-            print("Could not read error response body", file=sys.stderr)
-        sys.exit(1)
-    except Exception as e:
-        print(f"Error querying Overpass API: {e}", file=sys.stderr)
+            if attempt < max_attempts and e.code in (429, 504):
+                wait = 30 * attempt
+                print(f"Retrying in {wait}s...", file=sys.stderr)
+                time.sleep(wait)
+            else:
+                sys.exit(1)
+        except Exception as e:
+            print(f"Error querying Overpass API: {e}", file=sys.stderr)
         sys.exit(1)
 
 
 def convert_to_geojson(overpass_data):
     """Convert Overpass API response to GeoJSON format."""
     features = []
-    
+
     for element in overpass_data.get("elements", []):
-        if element["type"] == "way" and "geometry" in element:
-            coordinates = [[coord["lon"], coord["lat"]] for coord in element["geometry"]]
-            
+        if element["type"] == "node":
             feature = {
                 "type": "Feature",
                 "properties": element.get("tags", {}),
-                "geometry": {
-                    "type": "LineString",
-                    "coordinates": coordinates
-                }
-            }
-            features.append(feature)
-        
-        elif element["type"] == "node":
-            feature = {
-                "type": "Feature", 
-                "properties": element.get("tags", {}),
                 "geometry": {
                     "type": "Point",
                     "coordinates": [element["lon"], element["lat"]]
                 }
             }
             features.append(feature)
-    
+
+        elif element["type"] in ("way", "relation"):
+            if "geometry" in element:
+                # out geom; — full coordinate list (used for highways/paths)
+                coordinates = [[coord["lon"], coord["lat"]] for coord in element["geometry"]]
+                feature = {
+                    "type": "Feature",
+                    "properties": element.get("tags", {}),
+                    "geometry": {
+                        "type": "LineString",
+                        "coordinates": coordinates
+                    }
+                }
+                features.append(feature)
+            elif "center" in element:
+                # out center; — single centroid point (used for addresses on buildings)
+                c = element["center"]
+                feature = {
+                    "type": "Feature",
+                    "properties": element.get("tags", {}),
+                    "geometry": {
+                        "type": "Point",
+                        "coordinates": [c["lon"], c["lat"]]
+                    }
+                }
+                features.append(feature)
+
     return {
         "type": "FeatureCollection",
         "features": features