Create new project design plans

download-overpass.py

@@ -82,37 +82,53 @@ area(id:{area_id})->.searchArea;"""
     return query
 
 
-def query_overpass(query):
-    """Send query to Overpass API and return JSON response, with retries."""
-    url = "https://overpass-api.de/api/interpreter"
-    data = urllib.parse.urlencode({"data": query}).encode("utf-8")
-    req = urllib.request.Request(
-        url,
-        data=data,
-        headers={"Accept": "*/*", "User-Agent": "TheVillagesImport/1.0"},
-    )
+# Public Overpass API mirrors, tried in order when the primary is overloaded.
+OVERPASS_ENDPOINTS = [
+    "https://overpass-api.de/api/interpreter",
+    "https://overpass.kumi.systems/api/interpreter",
+    "https://overpass.private.coffee/api/interpreter",
+]
 
-    max_attempts = 3
-    for attempt in range(1, max_attempts + 1):
-        try:
-            with urllib.request.urlopen(req, timeout=300) as response:
-                return json.loads(response.read().decode("utf-8"))
-        except urllib.error.HTTPError as e:
+
+def query_overpass(query, endpoints=None):
+    """Send query to Overpass API and return JSON response.
+
+    Tries each endpoint in turn (with a couple of retries per endpoint on
+    429/504) before falling through to the next mirror. Overpass instances
+    share the underlying database, so any mirror returns equivalent data.
+    """
+    endpoints = endpoints or OVERPASS_ENDPOINTS
+    data = urllib.parse.urlencode({"data": query}).encode("utf-8")
+
+    attempts_per_endpoint = 2
+    for endpoint in endpoints:
+        req = urllib.request.Request(
+            endpoint,
+            data=data,
+            headers={"Accept": "*/*", "User-Agent": "TheVillagesImport/1.0"},
+        )
+        for attempt in range(1, attempts_per_endpoint + 1):
             try:
-                error_body = e.read().decode("utf-8")
-            except Exception:
-                error_body = "(unreadable)"
-            print(f"HTTP Error {e.code} on attempt {attempt}: {e.reason}", file=sys.stderr)
-            print(f"Error response body: {error_body}", file=sys.stderr)
-            if attempt < max_attempts and e.code in (429, 504):
-                wait = 30 * attempt
-                print(f"Retrying in {wait}s...", file=sys.stderr)
-                time.sleep(wait)
-            else:
-                sys.exit(1)
-        except Exception as e:
-            print(f"Error querying Overpass API: {e}", file=sys.stderr)
-        sys.exit(1)
+                print(f"Querying {endpoint} (attempt {attempt})...", file=sys.stderr)
+                with urllib.request.urlopen(req, timeout=300) as response:
+                    return json.loads(response.read().decode("utf-8"))
+            except urllib.error.HTTPError as e:
+                try:
+                    error_body = e.read().decode("utf-8")
+                except Exception:
+                    error_body = "(unreadable)"
+                print(f"HTTP Error {e.code} on {endpoint} attempt {attempt}: {e.reason}", file=sys.stderr)
+                print(f"Error response body: {error_body}", file=sys.stderr)
+                if attempt < attempts_per_endpoint and e.code in (429, 504):
+                    wait = 30 * attempt
+                    print(f"Retrying in {wait}s...", file=sys.stderr)
+                    time.sleep(wait)
+            except Exception as e:
+                print(f"Error querying {endpoint}: {e}", file=sys.stderr)
+                break  # non-HTTP error (timeout, connection refused): skip to next mirror
+
+    print("All Overpass endpoints failed.", file=sys.stderr)
+    sys.exit(1)
 
 
 def convert_to_geojson(overpass_data):
@@ -200,7 +216,13 @@ def main():
         default="highways",
         help="Type of data to download (default: highways)"
     )
-    
+    parser.add_argument(
+        "--endpoint", "-e",
+        action="append",
+        help="Overpass API endpoint to use (repeatable; tried in order). "
+             f"Default: {', '.join(OVERPASS_ENDPOINTS)}"
+    )
+
     args = parser.parse_args()
     
     # Create output directory if it doesn't exist
@@ -212,7 +234,7 @@ def main():
     # Build and execute query
     query = build_overpass_query(args.county, args.state, args.type)
     print(f"Query: {query}")
-    overpass_data = query_overpass(query)
+    overpass_data = query_overpass(query, endpoints=args.endpoint)
     
     # Convert to GeoJSON
     geojson = convert_to_geojson(overpass_data)