Speed up address and road diffs from hours to seconds, gzip GeoJSON downloads, and render map points on canvas with viewport-only OSM/county layers

web/server.py

@@ -3,7 +3,8 @@
 Flask web server for The Villages Import Tools
 """
 from functools import wraps
-from flask import Flask, render_template, jsonify, request, send_from_directory, session, redirect, url_for, make_response
+from flask import Flask, render_template, jsonify, request, send_from_directory, send_file, session, redirect, url_for, make_response
+from werkzeug.utils import safe_join
 from werkzeug.security import generate_password_hash, check_password_hash
 import subprocess
 import os
@@ -14,6 +15,7 @@ import shutil
 import threading
 import json
 import base64
+import gzip
 import logging
 from datetime import datetime, timezone
 import yaml
@@ -777,6 +779,9 @@ def clear_data():
                 try:
                     os.remove(fpath)
                     removed.append(f'{county["id"]}/{fname}')
+                    gz_copy = os.path.join(GZIP_CACHE_DIR, county['id'], fname + '.gz')
+                    if os.path.exists(gz_copy):
+                        os.remove(gz_copy)
                 except Exception as e:
                     errors.append(str(e))
     if errors:
@@ -820,9 +825,34 @@ def list_files():
 
     return jsonify(files)
 
+GZIP_CACHE_DIR = '/data/.gzcache'
+_gzip_lock = threading.Lock()
+
+
+def _gzipped_copy(path, filename):
+    """Path of a gzipped copy of path, (re)built when the source is newer.
+    GeoJSON compresses ~10x, which matters for 50-150 MB county files."""
+    gz_path = os.path.join(GZIP_CACHE_DIR, filename + '.gz')
+    with _gzip_lock:
+        if not os.path.exists(gz_path) or os.path.getmtime(gz_path) < os.path.getmtime(path):
+            os.makedirs(os.path.dirname(gz_path), exist_ok=True)
+            tmp = gz_path + '.tmp'
+            with open(path, 'rb') as src, gzip.open(tmp, 'wb', compresslevel=5) as dst:
+                shutil.copyfileobj(src, dst, 1024 * 1024)
+            os.replace(tmp, gz_path)
+    return gz_path
+
+
 @app.route('/data/<path:filename>')
 def serve_data(filename):
-    """Serve files from /data"""
+    """Serve files from /data (GeoJSON gzipped when the browser accepts it)"""
+    path = safe_join('/data', filename)
+    if (path and filename.endswith('.geojson') and os.path.isfile(path)
+            and 'gzip' in request.headers.get('Accept-Encoding', '')):
+        resp = make_response(send_file(_gzipped_copy(path, filename), mimetype='application/geo+json'))
+        resp.headers['Content-Encoding'] = 'gzip'
+        resp.headers['Vary'] = 'Accept-Encoding'
+        return resp
     return send_from_directory('/data', filename)
 
 # ── Config file download / upload ──────────────────────────────────────────────