get address efficiency by getting only centroids

download-overpass.py

@@ -7,8 +7,6 @@ Usage:
     python download-overpass.py --type addresses "Lake County" "Florida" output/addresses.geojson
     python download-overpass.py --type multimodal "Sumter County" "Florida" output/paths.geojson
 
-TODO:
-- Don't just download roads. Probably ignore relations also.
 """
 
 import argparse
@@ -49,7 +47,17 @@ def build_overpass_query(county_name, state_name, data_type="highways"):
     """Build Overpass API query for specified data type in a county."""
     area_id = get_county_area_id(county_name, state_name)
 
-    base_query = f"""[out:json][timeout:60];
+    # Addresses: include nwr so building ways and relations are captured too,
+    # but use "out center;" instead of "out geom;" so ways/relations return a
+    # single centroid rather than their full coordinate list — much smaller payload.
+    if data_type == "addresses":
+        query = f"""[out:json][timeout:180];
+area(id:{area_id})->.searchArea;
+nwr["addr:housenumber"](area.searchArea);
+out center;"""
+        return query
+
+    base_query = f"""[out:json][timeout:120];
 area(id:{area_id})->.searchArea;"""
 
     if data_type == "highways":
@@ -65,8 +73,6 @@ area(id:{area_id})->.searchArea;"""
         selector += 'way["highway"="service"](area.searchArea);'
         selector += 'way["highway"="track"](area.searchArea);'
         selector += ');'
-    elif data_type == "addresses":
-        selector = 'nwr["addr:housenumber"](area.searchArea);'
     elif data_type == "multimodal":
         selector = '(way["highway"="path"](area.searchArea);way["highway"="cycleway"](area.searchArea););'
     else:
@@ -77,21 +83,27 @@ area(id:{area_id})->.searchArea;"""
 
 
 def query_overpass(query):
-    """Send query to Overpass API and return JSON response."""
+    """Send query to Overpass API and return JSON response, with retries."""
     url = "https://overpass-api.de/api/interpreter"
     data = urllib.parse.urlencode({"data": query}).encode("utf-8")
 
-    
+    max_attempts = 3
+    for attempt in range(1, max_attempts + 1):
         try:
-        with urllib.request.urlopen(url, data=data) as response:
+            with urllib.request.urlopen(url, data=data, timeout=300) as response:
                 return json.loads(response.read().decode("utf-8"))
         except urllib.error.HTTPError as e:
-        print(f"HTTP Error {e.code}: {e.reason}", file=sys.stderr)
             try:
                 error_body = e.read().decode("utf-8")
+            except Exception:
+                error_body = "(unreadable)"
+            print(f"HTTP Error {e.code} on attempt {attempt}: {e.reason}", file=sys.stderr)
             print(f"Error response body: {error_body}", file=sys.stderr)
-        except:
-            print("Could not read error response body", file=sys.stderr)
+            if attempt < max_attempts and e.code in (429, 504):
+                wait = 30 * attempt
+                print(f"Retrying in {wait}s...", file=sys.stderr)
+                time.sleep(wait)
+            else:
                 sys.exit(1)
         except Exception as e:
             print(f"Error querying Overpass API: {e}", file=sys.stderr)
@@ -103,9 +115,21 @@ def convert_to_geojson(overpass_data):
     features = []
 
     for element in overpass_data.get("elements", []):
-        if element["type"] == "way" and "geometry" in element:
-            coordinates = [[coord["lon"], coord["lat"]] for coord in element["geometry"]]
+        if element["type"] == "node":
+            feature = {
+                "type": "Feature",
+                "properties": element.get("tags", {}),
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [element["lon"], element["lat"]]
+                }
+            }
+            features.append(feature)
 
+        elif element["type"] in ("way", "relation"):
+            if "geometry" in element:
+                # out geom; — full coordinate list (used for highways/paths)
+                coordinates = [[coord["lon"], coord["lat"]] for coord in element["geometry"]]
                 feature = {
                     "type": "Feature",
                     "properties": element.get("tags", {}),
@@ -115,14 +139,15 @@ def convert_to_geojson(overpass_data):
                     }
                 }
                 features.append(feature)
-        
-        elif element["type"] == "node":
+            elif "center" in element:
+                # out center; — single centroid point (used for addresses on buildings)
+                c = element["center"]
                 feature = {
                     "type": "Feature",
                     "properties": element.get("tags", {}),
                     "geometry": {
                         "type": "Point",
-                    "coordinates": [element["lon"], element["lat"]]
+                        "coordinates": [c["lon"], c["lat"]]
                     }
                 }
                 features.append(feature)