allow username/password encoded in SWUPD_VERSION_URL and SWUPD_CONTENT_URL

This patch allows basic authentication of swupd SWUPD_VERSION_URL and SWUPD_CONTENT_URL.
swupd-client already support urlencoded username/password, but buildlayer does not.

requested patch attached

From f2526a7ed47b3f3c8f0cb893eadb5e6981255d4c Mon Sep 17 00:00:00 2001
From: ingo <[email protected]>
Date: Sat, 25 Mar 2017 21:13:33 +0100
Subject: [PATCH] bundles.py: allow username/password encoded into HTTP server
 URLs example: https://user:password@server/path

---
 lib/swupd/bundles.py | 20 ++++++++++++++++++++

 1 file changed, 20 insertions(+)

lib/swupd/bundles.py +0 -0

index b4c6f49..223fd3c 100644
@@ -4,6 +4,8 @@ import subprocess

 import shutil
 import urllib.request
 import urllib.error

lib/swupd/bundles.py +2 -0

+import urllib.parse
+import re

 from bb.utils import export_proxies
 from oe.package_manager import RpmPM
 from oe.package_manager import OpkgPM

lib/swupd/bundles.py +0 -0

@@ -164,6 +166,15 @@ def download_manifests(content_url, version,

component, to_dir):
     base_versions = set()

     if not os.path.exists(target):
         bb.debug(1, 'Downloading %s -> %s' % (source, target)) +        parsed_source = urllib.parse.urlsplit(source)
+        if( parsed_source.username != None):
+            new_source = ( parsed_source.scheme, re.sub( re.escape( parsed_source.username+':'+parsed_source.password+'@'), '',parsed_source.netloc), parsed_source.path, parsed_source.query, parsed_source.fragment)

+            source = urllib.parse.urlunsplit( new_source) +            manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+            manager.add_password(None, new_source, parsed_source.username, parsed_source.password)
+            authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+            opener = urllib.request.build_opener(authHandler) +            urllib.request.install_opener(opener)

         response = urllib.request.urlopen(source) archive = response.read() bb.utils.mkdirhier(to_dir)

lib/swupd/bundles.py +0 -0

@@ -228,6 +239,15 @@ def download_old_versions(d):

     for format in range(3, current_format + 1):
         try:
             url = '%s/version/format%d/latest' % (content_url, format)
+            parsed_url = urllib.parse.urlsplit(url)
+            if( parsed_url.username != None):
+                new_url = ( parsed_url.scheme, re.sub( re.escape( parsed_url.username+':'+parsed_url.password+'@'), '',parsed_url.netloc), parsed_url.path, parsed_url.query, parsed_url.fragment)
+                url = urllib.parse.urlunsplit( new_url)

+                manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+                manager.add_password(None, new_url, parsed_url.username, parsed_url.password)
+                authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+                opener = urllib.request.build_opener(authHandler) +                urllib.request.install_opener(opener)

Downloading content and version information via HTTP may need a username/password for basic authentication. To support this, SWUPD_VERSION_URL and SWUPD_CONTENT_URL can now contain URLs of the form http(s)://<user>:<password>@<host>/.

Original patch from: Ingo Flaschberger <[email protected]>

Signed-off-by Patrick Ohly <[email protected]>

---
  lib/swupd/bundles.py | 26 ++++++++++++++++++++++++--
  1 file changed, 24 insertions(+), 2 deletions(-)

lib/swupd/bundles.py +24 -2

index e1eec5a..48c7455 100644
@@ -5,6 +5,7 @@ import subprocess
  import shutil
  import urllib.request
  import urllib.error
+import urllib.parse
  from bb.utils import export_proxies
  from oe.package_manager import RpmPM
  from oe.package_manager import OpkgPM

@@ -153,6 +154,27 @@ def copy_bundle_contents(d):
      for bndl in bundles:
          stage_empty_bundle(d, bndl)
 
+def handle_plain_auth(url):
+    """
+    Check for special urls with username/password (as in http://user:password@host/),
+    extract those and install an auth handler which will provide them
+    to the HTTP server when needed. Returns the URL that is to be instead of the original one.
+    """
+    parsed_url = urllib.parse.urlsplit(url)
+    if parsed_url.username != None:
+        # Use the netloc with just the hostname, without username/password.
+        parsed_url.netloc = parsed_url.hostname
+        # The username/password are installed permanently in the urllib.request module
+        # for future use with all URLs beneath url.
+        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+        manager.add_password(None, parsed_url, parsed_url.username, parsed_url.password)
+        authHandler = urllib.request.HTTPBasicAuthHandler(manager)
+        opener = urllib.request.build_opener(authHandler)
+        urllib.request.install_opener(opener)
+        return urllib.parse.urlunsplit(new_source)
+    else:
+        return url
+
  def download_manifests(content_url, version, component, to_dir):
      """
      Download one manifest file and recursively all manifests referenced by it.

@@ -204,8 +226,8 @@ def download_old_versions(d):
      a normal build and thus is not on the critical path.
      """
 
-    content_url = d.getVar('SWUPD_CONTENT_BUILD_URL', True)
-    version_url = d.getVar('SWUPD_VERSION_BUILD_URL', True)
+    content_url = handle_plain_auth(d.getVar('SWUPD_CONTENT_BUILD_URL', True))
+    version_url = handle_plain_auth(d.getVar('SWUPD_VERSION_BUILD_URL', True))
      current_format = int(d.getVar('SWUPD_FORMAT', True))
      deploy_dir = d.getVar('DEPLOY_DIR_SWUPD', True)
      www_dir = os.path.join(deploy_dir, 'www')

Dear Patrick,

this doesn't work:
      0162:    """
      0163:    parsed_url = urllib.parse.urlsplit(url) 0164:    if parsed_url.username != None:
      0165:        # Use the netloc with just the hostname, without username/password.
  *** 0166:        parsed_url.netloc = parsed_url.hostname 0167:        # The username/password are installed permanently in
the urllib.request module
      0168:        # for future use with all URLs beneath url. 0169:        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
      0170:        manager.add_password(None, parsed_url, parsed_url.username, parsed_url.password)
Exception: AttributeError: can't set attribute

Using hostname as netloc will also remove an additional portnumber - could this be a problem?

Quoted 35 lines

Downloading content and version information via HTTP may need a username/password for basic authentication. To support this, SWUPD_VERSION_URL and SWUPD_CONTENT_URL can now contain URLs of the form http(s)://<user>:<password>@<host>/.

Original patch from: Ingo Flaschberger <[email protected]

Quoted 1 line

Signed-off-by Patrick Ohly <[email protected]>

---
  lib/swupd/bundles.py | 26 ++++++++++++++++++++++++--

  1 file changed, 24 insertions(+), 2 deletions(-)

lib/swupd/bundles.py +0 -0

index e1eec5a..48c7455 100644
@@ -5,6 +5,7 @@ import subprocess

  import shutil
  import urllib.request
  import urllib.error
+import urllib.parse
  from bb.utils import export_proxies
  from oe.package_manager import RpmPM
  from oe.package_manager import OpkgPM

lib/swupd/bundles.py +0 -0

@@ -153,6 +154,27 @@ def copy_bundle_contents(d):

      for bndl in bundles:
          stage_empty_bundle(d, bndl)

+def handle_plain_auth(url):
+    """
+    Check for special urls with username/password (as in http://us
er:password@host/),
+    extract those and install an auth handler which will provide them

Quoted 8 lines

+    to the HTTP server when needed. Returns the URL that is to be instead of the original one.
+    """
+    parsed_url = urllib.parse.urlsplit(url)
+    if parsed_url.username != None:
+        # Use the netloc with just the hostname, without username/password.
+        parsed_url.netloc = parsed_url.hostname

Quoted 10 lines

+        # The username/password are installed permanently in the urllib.request module
+        # for future use with all URLs beneath url.
+        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm() +        manager.add_password(None, parsed_url, parsed_url.username, parsed_url.password)
+        authHandler = urllib.request.HTTPBasicAuthHandler(manager) +        opener = urllib.request.build_opener(authHandler) +        urllib.request.install_opener(opener)
+        return urllib.parse.urlunsplit(new_source)

Quoted 18 lines

Dear Patrick,

this doesn't work:
      0162:    """
      0163:    parsed_url = urllib.parse.urlsplit(url) 0164:    if parsed_url.username != None:
      0165:        # Use the netloc with just the hostname, without
username/password.
  *** 0166:        parsed_url.netloc = parsed_url.hostname 0167:        # The username/password are installed permanently
in
the urllib.request module
      0168:        # for future use with all URLs beneath url. 0169:        manager =
urllib.request.HTTPPasswordMgrWithDefaultRealm()
      0170:        manager.add_password(None, parsed_url,
parsed_url.username, parsed_url.password)
Exception: AttributeError: can't set attribute

Looks like netloc is a read-only attribute. That means one has to construct a new urllib.parse.SplitResult instead of updating the old one.

Quoted 2 lines

Using hostname as netloc will also remove an additional portnumber - could this be a problem?

Yes, that's also something that needs to be fixed.

Can you update the patch as shown below, test it, and it if works send the final version to the list? Obviously I am not doing a good job with posting code that I can't test :-/

Am 20.12.2017 um 16:50 schrieb Patrick Ohly:

Quoted 27 lines

Quoted 27 lines

Downloading content and version information via HTTP may need a username/password for basic authentication. To support this, SWUPD_VERSION_URL and SWUPD_CONTENT_URL can now contain URLs of the form http(s)://<user>:<password>@<host>/.

Original patch from: Ingo Flaschberger <[email protected]

Signed-off-by Patrick Ohly <[email protected]>

---
   lib/swupd/bundles.py | 26 ++++++++++++++++++++++++--
   1 file changed, 24 insertions(+), 2 deletions(-)

lib/swupd/bundles.py +1 -0

index e1eec5a..48c7455 100644
@@ -5,6 +5,7 @@ import subprocess
   import shutil
   import urllib.request
   import urllib.error
+import urllib.parse
   from bb.utils import export_proxies
   from oe.package_manager import RpmPM
   from oe.package_manager import OpkgPM

@@ -153,6 +154,27 @@ def copy_bundle_contents(d):
       for bndl in bundles:
           stage_empty_bundle(d, bndl)
 

Quoted 14 lines

Quoted 6 lines

lib/swupd/bundles.py +4 -0

+def handle_plain_auth(url):
+    """
+    Check for special urls with username/password (as in http://us
 er:password@host/),
+    extract those and install an auth handler which will provide

them

Quoted 8 lines

+    to the HTTP server when needed. Returns the URL that is to be instead of the original one.
+    """
+    parsed_url = urllib.parse.urlsplit(url)
+    if parsed_url.username != None:
+        # Use the netloc with just the hostname, without username/password.
+        parsed_url.netloc = parsed_url.hostname

Instead:

netloc = parsed_url.hostname
if parsed_url.port is not None:

     netloc += ":%d" % parsed_url.port

new_url = urllib.parse.SplitResult(parsed_url.scheme, netloc, parsed_url.path, parsed_url.query, parsed_url.fragment)

Quoted 10 lines

Quoted 10 lines

+        # The username/password are installed permanently in the urllib.request module
+        # for future use with all URLs beneath url.
+        manager = urllib.request.HTTPPasswordMgrWithDefaultRealm() +        manager.add_password(None, parsed_url, parsed_url.username, parsed_url.password)
+        authHandler = urllib.request.HTTPBasicAuthHandler(manager) +        opener = urllib.request.build_opener(authHandler) +        urllib.request.install_opener(opener)
+        return urllib.parse.urlunsplit(new_source)

Instead:

return urllib.parse.urlunsplit(new_url)

Dear Patrick,

it works if you replace:
     manager.add_password(None, parsed_url, parsed_url.username, parsed_url.password)
with:
      manager.add_password(None, new_url, parsed_url.username, parsed_url.password)

Quoted 8 lines

Dear Patrick,

it works if you replace:
     manager.add_password(None, parsed_url, parsed_url.username,
parsed_url.password)
with:
      manager.add_password(None, new_url, parsed_url.username,
parsed_url.password)

I assume this works on top of my proposal. To avoid such ambiguity and the risk that something gets merged that is still incomplete, please post the entire patch as tested by you. The right way would be "git send-email", but I can also take "git diff" attached to an email.