Skip to contentSkip to conversation

oeqa/ssh: Improve performance and log sizes

PATCH New Richard Purdie 1 participant 3 messages Nov 20, 2024

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 oeqa/ssh: Improve performance and log sizes

The current code is not fit for purpose when handling large files via ssh. In the strace ptest case, we can end up with a 1.4GB archive being transferred for which every byte is printed into the task logfile twice over. This is then sent over bitbake IPC which compounds the problems.

Make the following improvements:
* when the output is large (over 64kb), don't print it
* use a bytearray for better concat performance since strings are slow for this
* when there is no ssh output, say that
* print periodic size status output rather than the data itself since this could be binary and/or large
* fix the killed process message logic which appeared broken

---
 meta/lib/oeqa/core/target/ssh.py | 44 +++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 12 deletions(-)
meta/lib/oeqa/core/target/ssh.py +32 -12
index d4734693848..8b5c450a058 100644
@@ -92,7 +92,10 @@ class OESSHTarget(OETarget):
             processTimeout = self.timeout
 
         status, output = self._run(sshCmd, processTimeout, ignore_status, raw)
-        self.logger.debug('Command: %s\nStatus: %d Output:  %s\n' % (command, status, output))
+        if len(output) > (64 * 1024):
+            self.logger.debug('Command: %s\nStatus: %d Output length:  %s\n' % (command, status, len(output)))
+        else:
+            self.logger.debug('Command: %s\nStatus: %d Output:  %s\n' % (command, status, output))
 
         return (status, output)
 
@@ -211,17 +214,18 @@ def SSHCall(command, logger, timeout=None, raw=False, **opts):
     def run():
         nonlocal output
         nonlocal process
-        output_raw = b''
+        output_raw = bytearray()
         starttime = time.time()
+        progress = time.time()
         process = subprocess.Popen(command, **options)
         has_timeout = False
+        appendline = None
         if timeout:
             endtime = starttime + timeout
             eof = False
             os.set_blocking(process.stdout.fileno(), False)
             while not has_timeout and not eof:
                 try:
-                    logger.debug('Waiting for process output: time: %s, endtime: %s' % (time.time(), endtime))
                     if select.select([process.stdout], [], [], 5)[0] != []:
                         # wait a bit for more data, tries to avoid reading single characters
                         time.sleep(0.2)
@@ -229,9 +233,9 @@ def SSHCall(command, logger, timeout=None, raw=False, **opts):
                         if not data:
                             eof = True
                         else:
-                            output_raw += data
+                            output_raw.extend(data)
                             # ignore errors to capture as much as possible
-                            logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore'))
+                            #logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore'))
                             endtime = time.time() + timeout
                 except InterruptedError:
                     logger.debug('InterruptedError')
@@ -244,6 +248,10 @@ def SSHCall(command, logger, timeout=None, raw=False, **opts):
                     logger.debug('SSHCall has timeout! Time: %s, endtime: %s' % (time.time(), endtime))
                     has_timeout = True
 
+                if time.time() >= (progress + 60):
+                    logger.debug('Waiting for process output at time: %s with datasize: %s' % (time.time(), len(output_raw)))
+                    progress = time.time()
+
             process.stdout.close()
 
             # process hasn't returned yet
@@ -256,17 +264,29 @@ def SSHCall(command, logger, timeout=None, raw=False, **opts):
                     logger.debug('OSError when killing process')
                     pass
                 endtime = time.time() - starttime
-                lastline = ("\nProcess killed - no output for %d seconds. Total"
+                appendline = ("\nProcess killed - no output for %d seconds. Total"
                             " running time: %d seconds." % (timeout, endtime))
-                logger.debug('Received data from SSH call:\n%s ' % lastline)
-                output += lastline
+                logger.debug('Received data from SSH call:\n%s ' % appendline)
                 process.wait()
 
+            if raw:
+                output = bytes(output_raw)
+                if appendline:
+                    output += bytes(appendline, "utf-8")
+            else:
+                output = output_raw.decode('utf-8', errors='ignore')
+                if appendline:
+                    output += appendline
         else:
-            output_raw = process.communicate()[0]
-
-        output = output_raw if raw else output_raw.decode('utf-8', errors='ignore')
-        logger.debug('Data from SSH call:\n%s' % output.rstrip())
+            output = output_raw = process.communicate()[0]
+            if not raw:
+                output = output_raw.decode('utf-8', errors='ignore')
+
+        if len(output) < (64 * 1024):
+            if output.rstrip():
+                logger.debug('Data from SSH call:\n%s' % output.rstrip())
+            else:
+                logger.debug('No output from SSH call')
 
         # timout or not, make sure process exits and is not hanging
         if process.returncode == None:
#1 lore raw

PATCH 2/3 testimage: Handle /var/log being volatile

When /var/log is volatile, capturing just the symlink isn't useful. Fix this.

---
 meta/classes-recipe/testimage.bbclass | 1 +
 1 file changed, 1 insertion(+)
meta/classes-recipe/testimage.bbclass +1 -0
index 055002d3fdc..e6e9e5345ac 100644
@@ -24,6 +24,7 @@ TESTIMAGE_AUTO ??= "0"
 
 TESTIMAGE_FAILED_QA_ARTIFACTS = "\
     ${localstatedir}/log \
+    ${localstatedir}/volatile/log \
     ${sysconfdir}/version \
     ${sysconfdir}/os-release"
 
#2 lore raw

PATCH 3/3 qemu/testimage: Ensure GNU tar is in test images

In reply to Richard Purdie

We've seen cases where a ptest (strace) has a 43GB sparse file in the test directory. busybox tar doesn't work well with this. The resulting 1.4GB archive takes hours to extract too.

Ensure tar is added to our full images and use the sparse option to collect files for debugging. This stops crazy build hangs.

---
 meta/conf/machine/include/qemu.inc | 3 +++
 meta/lib/oeqa/utils/postactions.py | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)
meta/conf/machine/include/qemu.inc +3 -0
index bb7aec76750..d7392d47623 100644
@@ -28,3 +28,6 @@ KERNEL_FEATURES:append:pn-linux-yocto = " features/nfsd/nfsd-enable.scc"
 KERNEL_FEATURES:append:pn-linux-yocto-rt = " features/nfsd/nfsd-enable.scc"
 
 IMAGE_CLASSES += "qemuboot"
+
+# Use gnu tar so we can handle sparse files in testimage debug archives
+MACHINE_ESSENTIAL_EXTRA_RDEPENDS += "tar"
meta/lib/oeqa/utils/postactions.py +2 -1
index 8f787838b9d..c69481db6cb 100644
@@ -64,7 +64,8 @@ def list_and_fetch_failed_tests_artifacts(d, tc, artifacts_list, outputdir):
         bb.warn("Could not load artifacts list, skip artifacts retrieval")
         return
     try:
-        cmd = "tar zcf - " + " ".join(artifacts_list)
+        # We need gnu tar for sparse files, not busybox
+        cmd = "tar --sparse -zcf - " + " ".join(artifacts_list)
         (status, output) = tc.target.run(cmd, raw = True)
         if status != 0 or not output:
             raise Exception("Error while fetching compressed artifacts")