Image Status check

app/services/image_vector.py

@@ -147,6 +147,20 @@ def to_pg(vector: Iterable[int]) -> str:
 # URL -> bytes
 # ---------------------------------------------------------------------------
 
+_RATE_LIMIT_BACKOFF_SECONDS = 5.0
+_RATE_LIMIT_BACKOFF_CAP_SECONDS = 15.0
+
+
+def _retry_after_seconds(header: Optional[str]) -> float:
+    """Seconds to wait after a 429: the server's Retry-After if it is a plain
+    number, capped so one hostile header cannot stall the worker."""
+    try:
+        seconds = float(header) if header else _RATE_LIMIT_BACKOFF_SECONDS
+    except (TypeError, ValueError):
+        seconds = _RATE_LIMIT_BACKOFF_SECONDS
+    return max(0.0, min(seconds, _RATE_LIMIT_BACKOFF_CAP_SECONDS))
+
+
 def download_image_bytes(url: str, timeout: Optional[float] = None) -> Optional[bytes]:
     """Fetch `url` as image bytes, or None.
 
@@ -165,13 +179,20 @@ def download_image_bytes(url: str, timeout: Optional[float] = None) -> Optional[
     same_site = f"{parsed.scheme}://{parsed.netloc}/" if parsed.netloc else None
     timeout = timeout or IMAGE_VECTOR_TIMEOUT_SECONDS
 
-    for referer in (same_site, None):
+    # Third attempt exists for one reason: a 429. Wikimedia served ~100 of the
+    # Own Products images in a row and then throttled the rest; those URLs are
+    # fine, the pace was not. Honour Retry-After (capped) and try once more.
+    for referer in (same_site, None, same_site):
         headers = {"User-Agent": _BROWSER_UA, "Accept": "image/*,*/*;q=0.8"}
         if referer:
             headers["Referer"] = referer
         resp = None
         try:
             resp = requests.get(url, headers=headers, timeout=timeout, stream=True)
+            if resp.status_code == 429:
+                logger.debug("HTTP 429 for %s - backing off", url)
+                time.sleep(_retry_after_seconds(resp.headers.get("retry-after")))
+                continue
             if resp.status_code != 200:
                 logger.debug("HTTP %s for %s", resp.status_code, url)
                 continue