Skip to contentSkip to conversation

Improve performance when reading stdin

PATCH Superseded Herbert Xu 1 participant 4 messages May 19, 2024

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Herbert Xu

author
#0 lore raw

v2 cover Improve performance when reading stdin

v2 rebases on the new multibyte patch-set.

The performance when reading stdin has regressed after we started reading one byte at a time. This patch series uses lseek(2) and tee(2) to recover performance where possible.

Herbert Xu (3):
input: Move newline loop into preadbuffer
input: Use lseek on stdin when possible
input: Use tee(2) for stdin pipe

 configure.ac  |   2 +-
 src/eval.c    |   3 +
 src/init.h    |   1 +
 src/input.c   | 208 +++++++++++++++++++++++++++++++++++++++-----------
 src/input.h   |   4 +
 src/jobs.c    |   3 +
 src/mkinit.c  |   6 ++
 src/options.c |   2 +-
 src/redir.c   |  26 ++++---
 src/system.h  |   7 ++
 src/trap.c    |   1 +
 11 files changed, 205 insertions(+), 58 deletions(-)
Signature · 1 line
2.39.2
author
#1 lore raw

PATCH v2 1/3 input: Move newline loop into preadbuffer

As it stands preadfd tries to fetch a whole line when we're reading one byte at a time. However, this is wrong because how many bytes we're trying to read has nothing to do with whether we get a whole line or not.

Move the loop into preadbuffer alongside the other history support code.

---
 src/input.c | 47 +++++++++++++++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 16 deletions(-)
src/input.c +31 -16
index 6779069..193235d 100644
@@ -192,16 +192,27 @@ preadfd(void)
 {
 	char *buf = parsefile->buf;
 	int unget;
+	int pnr;
 	int nr;
 
+	nr = input_get_lleft(parsefile);
+
 	unget = parsefile->nextc - buf;
 	if (unget > PUNGETC_MAX)
 		unget = PUNGETC_MAX;
 
-	memmove(buf, parsefile->nextc - unget, unget);
-	parsefile->nextc = buf += unget;
+	memmove(buf, parsefile->nextc - unget, unget + nr);
+	buf += unget;
+	parsefile->nextc = buf;
+	buf += nr;
 
+	nr = BUFSIZ - nr;
+	if (!IS_DEFINED_SMALL && !nr)
+		return nr;
+
+	pnr = nr;
 retry:
+	nr = pnr;
 #ifndef SMALL
 	if (parsefile->fd == 0 && el) {
 		static const char *rl_cp;
@@ -216,9 +227,8 @@ retry:
 		if (rl_cp == NULL)
 			nr = 0;
 		else {
-			nr = el_len;
-			if (nr > BUFSIZ)
-				nr = BUFSIZ;
+			if (nr > el_len)
+				nr = el_len;
 			memcpy(buf, rl_cp, nr);
 			if (nr != el_len) {
 				el_len -= nr;
@@ -230,10 +240,8 @@ retry:
 	} else
 #endif
 	if (parsefile->fd)
-		nr = read(parsefile->fd, buf, BUFSIZ);
+		nr = read(parsefile->fd, buf, nr);
 	else {
-		unsigned len = BUFSIZ;
-
 		nr = 0;
 
 		do {
@@ -255,7 +263,7 @@ retry:
 			}
 
 			nr++;
-		} while (!IS_DEFINED_SMALL && *buf++ != '\n' && --len);
+		} while (0);
 	}
 
 	if (nr < 0) {
@@ -290,19 +298,26 @@ static int preadbuffer(void)
 		return PEOF;
 	flushall();
 
+	q = parsefile->nextc;
+	something = !first;
+
 	more = input_get_lleft(parsefile);
 	if (more <= 0) {
+		int nr;
+
 again:
-		if ((more = preadfd()) <= 0) {
+		nr = q - parsefile->nextc;
+		more = preadfd();
+		q = parsefile->nextc + nr;
+		if (more <= 0) {
 			input_set_lleft(parsefile, parsefile->nleft = 0);
+			if (!IS_DEFINED_SMALL && nr > 0)
+				goto save;
 			return PEOF;
 		}
 	}
 
-	q = parsefile->nextc;
-
 	/* delete nul characters */
-	something = !first;
 	for (;;) {
 		int c;
 
@@ -321,7 +336,6 @@ again:
 
 		switch (c) {
 		case '\n':
-			parsefile->nleft = q - parsefile->nextc - 1;
 			goto done;
 
 		default:
@@ -335,8 +349,7 @@ again:
 
 check:
 		if (more <= 0) {
-			parsefile->nleft = q - parsefile->nextc - 1;
-			if (parsefile->nleft < 0)
+			if (!IS_DEFINED_SMALL)
 				goto again;
 			break;
 		}
@@ -344,6 +357,8 @@ check:
 done:
 	input_set_lleft(parsefile, more);
 
+save:
+	parsefile->nleft = q - parsefile->nextc - 1;
 	if (!IS_DEFINED_SMALL)
 		savec = *q;
 	*q = '\0';
Signature · 1 line
2.39.2
author
#2 lore raw

PATCH v2 2/3 input: Use lseek on stdin when possible

In reply to Herbert Xu

For files that can be sought, use lseek instead of reading one byte at a time.

---
 src/eval.c    |   3 ++
 src/init.h    |   1 +
 src/input.c   | 108 +++++++++++++++++++++++++++++++++++---------------
 src/input.h   |   4 ++
 src/jobs.c    |   3 ++
 src/mkinit.c  |   6 +++
 src/options.c |   2 +-
 src/redir.c   |  26 +++++++-----
 src/trap.c    |   1 +
 9 files changed, 111 insertions(+), 43 deletions(-)
src/eval.c +3 -0
index 32f1e64..c785edb 100644
@@ -896,6 +896,8 @@ bail:
 		goto bail;
 
 	default:
+		flush_input();
+
 		/* Fork off a child process if necessary. */
 		if (!(flags & EV_EXIT) || have_traps()) {
 			INTOFF;
@@ -1131,6 +1133,7 @@ execcmd(int argc, char **argv)
 		iflag = 0;		/* exit on error */
 		mflag = 0;
 		optschanged();
+		flush_input();
 		shellexec(argv + 1, pathval(), 0);
 	}
 	return 0;
src/init.h +1 -0
index 4f98b5d..e117895 100644
@@ -39,4 +39,5 @@ union node;
 void init(void);
 void exitreset(void);
 void forkreset(union node *);
+void postexitreset(void);
 void reset(void);
src/input.c +76 -32

196 lines not on this page · raw message

src/input.h +4 -0
index c59d784..af1c1be 100644
@@ -35,6 +35,7 @@
  */
 
 #include <limits.h>
+#include <stdbool.h>
 
 #ifdef SMALL
 #define IS_DEFINED_SMALL 1
@@ -94,6 +95,7 @@ struct parsefile {
 };
 
 extern struct parsefile *parsefile;
+extern int stdin_istty;
 
 /*
  * The input line number.  Input.c just defines this variable, and saves
@@ -113,6 +115,8 @@ void pushstdin(void);
 void popfile(void);
 void unwindfiles(struct parsefile *);
 void popallfiles(void);
+void flush_input(void);
+void reset_input(void);
 
 static inline int input_get_lleft(struct parsefile *pf)
 {
src/jobs.c +3 -0
index 4cf6b8c..5765e6d 100644
@@ -968,6 +968,9 @@ forkshell(struct job *jp, union node *n, int mode)
 	int pid;
 
 	TRACE(("forkshell(%%%d, %p, %d) called\n", jobno(jp), n, mode));
+
+	flush_input();
+
 	pid = fork();
 	if (pid == 0)
 		forkchild(jp, n, mode);
src/mkinit.c +6 -0
index 870b64d..2514ebf 100644
@@ -119,6 +119,11 @@ char forkreset[] = "\
  * This routine is called when we enter a subshell.\n\
  */\n";
 
+char postexitreset[] = "\
+/*\n\
+ * This routine is called in exitshell.\n\
+ */\n";
+
 char reset[] = "\
 /*\n\
  * This routine is called when an error or an interrupt occurs in an\n\
@@ -130,6 +135,7 @@ struct event event[] = {
 	{"INIT", "init", init},
 	{"EXITRESET", "exitreset", exitreset},
 	{"FORKRESET", "forkreset", forkreset, "union node *n"},
+	{"POSTEXITRESET", "postexitreset", postexitreset},
 	{"RESET", "reset", reset},
 	{NULL, NULL}
 };
src/options.c +1 -1
index f157321..4d0a53a 100644
@@ -142,7 +142,7 @@ procargs(int argc, char **argv)
 			sh_error("-c requires an argument");
 		sflag = 1;
 	}
-	if (iflag == 2 && sflag == 1 && isatty(0) && isatty(1))
+	if (iflag == 2 && sflag == 1 && stdin_istty && isatty(1))
 		iflag = 1;
 	if (mflag == 2)
 		mflag = iflag;
src/redir.c +16 -10
index c57d745..8d1c8f6 100644
@@ -46,16 +46,17 @@
  * Code for dealing with input/output redirection.
  */
 
-#include "main.h"
-#include "shell.h"
-#include "nodes.h"
-#include "jobs.h"
-#include "options.h"
-#include "expand.h"
-#include "redir.h"
-#include "output.h"
-#include "memalloc.h"
 #include "error.h"
+#include "expand.h"
+#include "input.h"
+#include "jobs.h"
+#include "main.h"
+#include "memalloc.h"
+#include "nodes.h"
+#include "options.h"
+#include "output.h"
+#include "redir.h"
+#include "shell.h"
 #include "system.h"
 #include "trap.h"
 
@@ -142,6 +143,8 @@ redirect(union node *redir, int flags)
 			continue;
 
 		fd = n->nfile.fd;
+		if (fd == 0)
+			reset_input();
 
 		if (sv) {
 			int closed;
@@ -415,8 +418,11 @@ popredir(int drop)
 				close(i);
 			break;
 		default:
-			if (!drop)
+			if (!drop) {
+				if (i == 0)
+					reset_input();
 				dup2(rp->renamed[i], i);
+			}
 			close(rp->renamed[i]);
 			break;
 		}
src/trap.c +1 -0
index f871656..2351b61 100644
@@ -426,6 +426,7 @@ exitshell(void)
 	}
 out:
 	exitreset();
+	postexitreset();
 	/*
 	 * Disable job control so that whoever had the foreground before we
 	 * started can get it back.
Signature · 1 line
2.39.2
author
#3 lore raw

PATCH v2 3/3 input: Use tee(2) for stdin pipe

In reply to Herbert Xu

use tee(2) to peek at pipes in order to avoid reading one byte at a time.

---
 configure.ac |  2 +-
 src/input.c  | 71 ++++++++++++++++++++++++++++++++++++++++++++++------
 src/system.h |  7 ++++++
 3 files changed, 72 insertions(+), 8 deletions(-)
configure.ac +1 -1
index cb55c3f..50effc0 100644
@@ -89,7 +89,7 @@ dnl Checks for library functions.
 AC_CHECK_FUNCS(bsearch faccessat getpwnam getrlimit isalpha killpg \
 	       memfd_create mempcpy \
 	       sigsetmask stpcpy strchrnul strsignal strtod strtoimax \
-	       strtoumax sysconf)
+	       strtoumax sysconf tee)
 
 dnl Check whether it's worth working around FreeBSD PR kern/125009.
 dnl The traditional behavior of access/faccessat is crazy, but
src/input.c +64 -7

118 lines not on this page · raw message

src/system.h +7 -0
index 6b31d52..e7f968b 100644
@@ -118,6 +118,13 @@ long sysconf(int) __attribute__((__noreturn__));
 int isblank(int c);
 #endif
 
+#ifndef HAVE_TEE
+static inline ssize_t tee(int fd_in, int fd_out, size_t len, unsigned int flags)
+{
+	return -1;
+}
+#endif
+
 #ifndef HAVE_FNMATCH
 static inline int fnmatch(const char *pattern, const char *string, int flags)
 {
Signature · 1 line
2.39.2