fetch2/gcp: treat GatewayTimeout as fetch failure
PATCH 1/1 fetch2/gcp: treat GatewayTimeout as fetch failure
From: Sebastian Muxel <[email protected]>
blob.exists() and blob.download_to_filename() can raise google.api_core.exceptions.GatewayTimeout after the GCS client's own retries. Uncaught, that escapes as a hard error instead of a normal fetch/checkstatus failure (and blocks mirror fallback on download).
Catch GatewayTimeout in checkstatus() and download(), log a warning so the timeout is visible, and raise FetchError.
AI-Generated: Cursor with Grok 4.5
Signed-off-by Sebastian Muxel <[email protected]>
Signed-off-by Etienne Cordonnier <[email protected]>
--- lib/bb/fetch2/gcp.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-)
lib/bb/fetch2/gcp.py +22 -2
@@ -57,7 +57,7 @@ class GCP(FetchMethod): Fetch urls using the GCP API. Assumes localpath was called first. """ - from google.api_core.exceptions import NotFound + from google.api_core.exceptions import GatewayTimeout, NotFound logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}") if self.gcp_client is None: self.get_gcp_client()
@@ -71,6 +71,13 @@ class GCP(FetchMethod): blob.download_to_filename(ud.localpath) except NotFound: raise FetchError("The GCP API threw a NotFound exception") + except GatewayTimeout as e: + # The GCS client already retries GatewayTimeout internally. + # Raise FetchError so mirror fallback can proceed. + logger.warning( + f"GCP API GatewayTimeout while downloading gs://{ud.host}{ud.path}: {e}" + ) + raise FetchError(f"Transient GCP API GatewayTimeout for gs://{ud.host}{ud.path}") # Additional sanity checks copied from the wget class (although there # are no known issues which mean these are required, treat the GCP API
@@ -88,6 +95,8 @@ class GCP(FetchMethod): """ Check the status of a URL. """ + from google.api_core.exceptions import GatewayTimeout + logger.debug2(f"Checking status of gs://{ud.host}{ud.path}") if self.gcp_client is None: self.get_gcp_client()
@@ -96,7 +105,18 @@ class GCP(FetchMethod): # Path sometimes has leading slash, so strip it path = ud.path.lstrip("/") - if self.gcp_client.bucket(ud.host).blob(path).exists() == False: + try: + exists = self.gcp_client.bucket(ud.host).blob(path).exists() + except GatewayTimeout as e: + # The GCS client already retries GatewayTimeout internally. + # Surface a normal checkstatus failure and warn so the timeout + # is visible to operators. + logger.warning( + f"GCP API GatewayTimeout while checking gs://{ud.host}{ud.path}; treating as unavailable: {e}" + ) + raise FetchError(f"Transient GCP API GatewayTimeout for gs://{ud.host}{ud.path}") + + if exists == False: raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist") else: return True
Signature · 1 line
2.43.0