bitbake: fetch2/wget: checkstatus: cleanup URLError exception
PATCH v2 1/2 bitbake: fetch2/wget: checkstatus: cleanup URLError exception
The exception for urllib.error.URLError can only happen on the
with opener.open(r) as response:
line.
Limit the try except span to this line and move the retry logic of commit 312f1a5e741 outside of the try/except catch
see also commit:
312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails
Signed-off-by Nicola Lunghi <[email protected]>
--- bitbake/lib/bb/fetch2/wget.py | 60 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 31 deletions(-)
bitbake/lib/bb/fetch2/wget.py +29 -31
@@ -295,42 +295,40 @@ class Wget(FetchMethod): handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) opener = urllib.request.build_opener(*handlers) - try: - uri = ud.url.split(";")[0] - r = urllib.request.Request(uri) - r.get_method = lambda: "HEAD" - # Some servers (FusionForge, as used on Alioth) require that the - # optional Accept header is set. - r.add_header("Accept", "*/*") - def add_basic_auth(login_str, request): - '''Adds Basic auth to http request, pass in login:password as string''' - import base64 - encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") - authheader = "Basic %s" % encodeuser - r.add_header("Authorization", authheader) - - if ud.user and ud.pswd: - add_basic_auth(ud.user + ':' + ud.pswd, r) + uri = ud.url.split(";")[0] + r = urllib.request.Request(uri) + r.get_method = lambda: "HEAD" + # Some servers (FusionForge, as used on Alioth) require that the + # optional Accept header is set. + r.add_header("Accept", "*/*") + def add_basic_auth(login_str, request): + '''Adds Basic auth to http request, pass in login:password as string''' + import base64 + encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") + authheader = "Basic %s" % encodeuser + r.add_header("Authorization", authheader) - try: - import netrc - n = netrc.netrc() - login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) - add_basic_auth("%s:%s" % (login, password), r) - except (TypeError, ImportError, IOError, netrc.NetrcParseError): - pass + if ud.user and ud.pswd: + add_basic_auth(ud.user + ':' + ud.pswd, r) + try: + import netrc + n = netrc.netrc() + login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) + add_basic_auth("%s:%s" % (login, password), r) + except (TypeError, ImportError, IOError, netrc.NetrcParseError): + pass + + try: with opener.open(r) as response: pass + return True except urllib.error.URLError as e: - if try_again: - logger.debug(2, "checkstatus: trying again") - return self.checkstatus(fetch, ud, d, False) - else: - # debug for now to avoid spamming the logs in e.g. remote sstate searches - logger.debug(2, "checkstatus() urlopen failed: %s" % e) - return False - return True + logger.debug(2, "checkstatus() urlopen failed: %s" % e) + if try_again: + logger.debug(2, "checkstatus: trying again") + return self.checkstatus(fetch, ud, d, False) + return False def _parse_path(self, regex, s): """
Signature · 1 line
2.20.1
PATCH v2 2/2 bitbake: fetch2/wget: checkstatus: use the same arguments as parent
the commit:
312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails
was introducing an extra parameter to the checkstatus function, overriding the parent.
Avoid that and remove recursion as is not needed. Also introduce a small delay between the two requests.
Signed-off-by Nicola Lunghi <[email protected]>
--- bitbake/lib/bb/fetch2/wget.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-)
bitbake/lib/bb/fetch2/wget.py +16 -5
@@ -121,7 +121,7 @@ class Wget(FetchMethod): return True - def checkstatus(self, fetch, ud, d, try_again=True): + def checkstatus(self, fetch, ud, d): class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self):
@@ -324,10 +324,21 @@ class Wget(FetchMethod): pass return True except urllib.error.URLError as e: - logger.debug(2, "checkstatus() urlopen failed: %s" % e) - if try_again: - logger.debug(2, "checkstatus: trying again") - return self.checkstatus(fetch, ud, d, False) + logger.debug(2, "checkstatus() urlopen failed: %s", e) + + # Some services such as SourceForge seem to struggle to keep up under + # load, with the result that over half of the autobuilder checkuri + # runs fail with sourceforge.net "connection timed out". Attempt to + # mitigate this by re-attempting once the network operation on failure. + import time + time.sleep(0.2) + logger.debug(2, "checkstatus: trying again") + try: + with opener.open(r) as response: + pass + return True + except urllib.error.URLError as e: + logger.debug(2, "checkstatus() urlopen failed for the second time: %s", e) return False def _parse_path(self, regex, s):
Signature · 1 line
2.20.1