Skip to contentSkip to conversation

bitbake: fetch2/wget: checkstatus: cleanup URLError exception

PATCH Superseded Nicola Lunghi 1 participant 2 messages Dec 19, 2019

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 bitbake: fetch2/wget: checkstatus: cleanup URLError exception

The exception for urllib.error.URLError can only happen on the

with opener.open(r) as response:

line.
Limit the try except span to this line and move the retry logic of commit 312f1a5e741 outside of the try/except catch

see also commit:
312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails

---
 bitbake/lib/bb/fetch2/wget.py | 60 +++++++++++++++++------------------
 1 file changed, 29 insertions(+), 31 deletions(-)
bitbake/lib/bb/fetch2/wget.py +29 -31
index 90aa9b190a..180fe9aacd 100644
@@ -375,42 +375,40 @@ class Wget(FetchMethod):
             handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context()))
         opener = urllib.request.build_opener(*handlers)
 
-        try:
-            uri = ud.url.split(";")[0]
-            r = urllib.request.Request(uri)
-            r.get_method = lambda: "HEAD"
-            # Some servers (FusionForge, as used on Alioth) require that the
-            # optional Accept header is set.
-            r.add_header("Accept", "*/*")
-            def add_basic_auth(login_str, request):
-                '''Adds Basic auth to http request, pass in login:password as string'''
-                import base64
-                encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
-                authheader = "Basic %s" % encodeuser
-                r.add_header("Authorization", authheader)
-
-            if ud.user and ud.pswd:
-                add_basic_auth(ud.user + ':' + ud.pswd, r)
+        uri = ud.url.split(";")[0]
+        r = urllib.request.Request(uri)
+        r.get_method = lambda: "HEAD"
+        # Some servers (FusionForge, as used on Alioth) require that the
+        # optional Accept header is set.
+        r.add_header("Accept", "*/*")
+        def add_basic_auth(login_str, request):
+            '''Adds Basic auth to http request, pass in login:password as string'''
+            import base64
+            encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
+            authheader = "Basic %s" % encodeuser
+            r.add_header("Authorization", authheader)
+
+        if ud.user and ud.pswd:
+            add_basic_auth(ud.user + ':' + ud.pswd, r)
 
-            try:
-                import netrc
-                n = netrc.netrc()
-                login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
-                add_basic_auth("%s:%s" % (login, password), r)
-            except (TypeError, ImportError, IOError, netrc.NetrcParseError):
-                pass
+        try:
+            import netrc
+            n = netrc.netrc()
+            login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
+            add_basic_auth("%s:%s" % (login, password), r)
+        except (TypeError, ImportError, IOError, netrc.NetrcParseError):
+            pass
 
+        try:
             with opener.open(r) as response:
                 pass
+            return True
         except urllib.error.URLError as e:
-            if try_again:
-                logger.debug(2, "checkstatus: trying again")
-                return self.checkstatus(fetch, ud, d, False)
-            else:
-                # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug(2, "checkstatus() urlopen failed: %s" % e)
-                return False
-        return True
+            logger.debug(2, "checkstatus() urlopen failed: %s" % e)
+        if try_again:
+            logger.debug(2, "checkstatus: trying again")
+            return self.checkstatus(fetch, ud, d, False)
+        return False
 
     def _parse_path(self, regex, s):
         """
Signature · 1 line
2.20.1
#1 lore raw

PATCH 2/2 bitbake: fetch2/wget: checkstatus: use the same arguments as parent

the commit:
312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails

was introducing an extra parameter to the checkstatus function, overriding the parent.

Avoid that and remove recursion as is not needed. Also introduce a small delay between the two requests.

---
 bitbake/lib/bb/fetch2/wget.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
bitbake/lib/bb/fetch2/wget.py +16 -5
index 180fe9aacd..9c0defb6d2 100644
@@ -201,7 +201,7 @@ class Wget(FetchMethod):
 
         return True
 
-    def checkstatus(self, fetch, ud, d, try_again=True):
+    def checkstatus(self, fetch, ud, d):
         class HTTPConnectionCache(http.client.HTTPConnection):
             if fetch.connection_cache:
                 def connect(self):
@@ -404,10 +404,21 @@ class Wget(FetchMethod):
                 pass
             return True
         except urllib.error.URLError as e:
-            logger.debug(2, "checkstatus() urlopen failed: %s" % e)
-        if try_again:
-            logger.debug(2, "checkstatus: trying again")
-            return self.checkstatus(fetch, ud, d, False)
+            logger.debug(2, "checkstatus() urlopen failed: %s", e)
+
+        # Some services such as SourceForge seem to struggle to keep up under
+        # load, with the result that over half of the autobuilder checkuri
+        # runs fail with sourceforge.net "connection timed out". Attempt to
+        # mitigate this by re-attempting once the network operation on failure.
+        import time
+        time.sleep(0.2)
+        logger.debug(2, "checkstatus: trying again")
+        try:
+            with opener.open(r) as response:
+                pass
+            return True
+        except urllib.error.URLError as e:
+            logger.debug(2, "checkstatus() urlopen failed for the second time: %s", e)
         return False
 
     def _parse_path(self, regex, s):
Signature · 1 line
2.20.1