bitbake: fetch2/wget: checkstatus: cleanup URLError exception
PATCH 1/2 bitbake: fetch2/wget: checkstatus: cleanup URLError exception
The exception for urllib.error.URLError can only happen on the
with opener.open(r) as response:
line.
Limit the try except span to this line and move the retry logic of commit 312f1a5e741 outside of the try/except catch
see also commit:
312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails
Change-Id: I72fd4dd663412660632e4c5d3bfac0f4282270e2
--- bitbake/lib/bb/fetch2/wget.py | 60 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 31 deletions(-)
bitbake/lib/bb/fetch2/wget.py +29 -31
@@ -375,42 +375,40 @@ class Wget(FetchMethod): handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) opener = urllib.request.build_opener(*handlers) - try: - uri = ud.url.split(";")[0] - r = urllib.request.Request(uri) - r.get_method = lambda: "HEAD" - # Some servers (FusionForge, as used on Alioth) require that the - # optional Accept header is set. - r.add_header("Accept", "*/*") - def add_basic_auth(login_str, request): - '''Adds Basic auth to http request, pass in login:password as string''' - import base64 - encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") - authheader = "Basic %s" % encodeuser - r.add_header("Authorization", authheader) - - if ud.user and ud.pswd: - add_basic_auth(ud.user + ':' + ud.pswd, r) + uri = ud.url.split(";")[0] + r = urllib.request.Request(uri) + r.get_method = lambda: "HEAD" + # Some servers (FusionForge, as used on Alioth) require that the + # optional Accept header is set. + r.add_header("Accept", "*/*") + def add_basic_auth(login_str, request): + '''Adds Basic auth to http request, pass in login:password as string''' + import base64 + encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") + authheader = "Basic %s" % encodeuser + r.add_header("Authorization", authheader) + + if ud.user and ud.pswd: + add_basic_auth(ud.user + ':' + ud.pswd, r) - try: - import netrc - n = netrc.netrc() - login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) - add_basic_auth("%s:%s" % (login, password), r) - except (TypeError, ImportError, IOError, netrc.NetrcParseError): - pass + try: + import netrc + n = netrc.netrc() + login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) + add_basic_auth("%s:%s" % (login, password), r) + except (TypeError, ImportError, IOError, netrc.NetrcParseError): + pass + try: with opener.open(r) as response: pass + return True except urllib.error.URLError as e: - if try_again: - logger.debug(2, "checkstatus: trying again") - return self.checkstatus(fetch, ud, d, False) - else: - # debug for now to avoid spamming the logs in e.g. remote sstate searches - logger.debug(2, "checkstatus() urlopen failed: %s" % e) - return False - return True + logger.debug(2, "checkstatus() urlopen failed: %s" % e) + if try_again: + logger.debug(2, "checkstatus: trying again") + return self.checkstatus(fetch, ud, d, False) + return False def _parse_path(self, regex, s): """
Signature · 1 line
2.20.1
PATCH 2/2 bitbake: fetch2/wget: checkstatus: use the same arguments as parent
the commit:
312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails
was introducing an extra parameter to the checkstatus function, overriding the parent.
Avoid that and remove recursion as is not needed. Also introduce a small delay between the two requests.
--- bitbake/lib/bb/fetch2/wget.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-)
bitbake/lib/bb/fetch2/wget.py +16 -5
@@ -201,7 +201,7 @@ class Wget(FetchMethod): return True - def checkstatus(self, fetch, ud, d, try_again=True): + def checkstatus(self, fetch, ud, d): class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self):
@@ -404,10 +404,21 @@ class Wget(FetchMethod): pass return True except urllib.error.URLError as e: - logger.debug(2, "checkstatus() urlopen failed: %s" % e) - if try_again: - logger.debug(2, "checkstatus: trying again") - return self.checkstatus(fetch, ud, d, False) + logger.debug(2, "checkstatus() urlopen failed: %s", e) + + # Some services such as SourceForge seem to struggle to keep up under + # load, with the result that over half of the autobuilder checkuri + # runs fail with sourceforge.net "connection timed out". Attempt to + # mitigate this by re-attempting once the network operation on failure. + import time + time.sleep(0.2) + logger.debug(2, "checkstatus: trying again") + try: + with opener.open(r) as response: + pass + return True + except urllib.error.URLError as e: + logger.debug(2, "checkstatus() urlopen failed for the second time: %s", e) return False def _parse_path(self, regex, s):
Signature · 1 line
2.20.1
✗ patchtest: failure for "bitbake: fetch2/wget: checksta..." and 1 more
In reply to Nicola Lunghi== Series Details ==
Series: "bitbake: fetch2/wget: checksta..." and 1 more
Revision: 1
URL : https://patchwork.openembedded.org/series/21688/
State : failure
== Summary ==
Thank you for submitting this patch series to OpenEmbedded Core. This is an automated response. Several tests have been executed on the proposed series by patchtest resulting in the following failures:
* Issue Series sent to the wrong mailing list or some patches from the series correspond to different mailing lists [test_target_mailing_list]
Suggested fix Send the series again to the correct mailing list (ML) Suggested ML [email protected] [http://git.openembedded.org/bitbake/] Patch's path: bitbake/lib/bb/fetch2/wget.py
* Issue Series does not apply on top of target branch [test_series_merge_on_head]
Suggested fix Rebase your series on top of targeted branch
Targeted branch master (currently at d6c62cc030)
If you believe any of these test results are incorrect, please reply to the mailing list ([email protected]) raising your concerns. Otherwise we would appreciate you correcting the issues and submitting a new version of the patchset if applicable. Please ensure you add/increment the version number when sending the new version (i.e. [PATCH] -> [PATCH v2] -> [PATCH v3] -> ...).
---
Guidelines: https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest Test suite: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe