Improve performance when reading stdin
v2 cover Improve performance when reading stdin
v2 rebases on the new multibyte patch-set.
The performance when reading stdin has regressed after we started reading one byte at a time. This patch series uses lseek(2) and tee(2) to recover performance where possible.
Herbert Xu (3):
input: Move newline loop into preadbuffer
input: Use lseek on stdin when possible
input: Use tee(2) for stdin pipe
configure.ac | 2 +- src/eval.c | 3 + src/init.h | 1 + src/input.c | 208 +++++++++++++++++++++++++++++++++++++++----------- src/input.h | 4 + src/jobs.c | 3 + src/mkinit.c | 6 ++ src/options.c | 2 +- src/redir.c | 26 ++++--- src/system.h | 7 ++ src/trap.c | 1 + 11 files changed, 205 insertions(+), 58 deletions(-)
Signature · 1 line
2.39.2
PATCH v2 1/3 input: Move newline loop into preadbuffer
As it stands preadfd tries to fetch a whole line when we're reading one byte at a time. However, this is wrong because how many bytes we're trying to read has nothing to do with whether we get a whole line or not.
Move the loop into preadbuffer alongside the other history support code.
--- src/input.c | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-)
src/input.c +31 -16
@@ -192,16 +192,27 @@ preadfd(void) {char *buf = parsefile->buf;int unget; +int pnr;int nr; + nr =input_get_lleft (parsefile); + unget = parsefile->nextc - buf;if (unget >PUNGETC_MAX ) unget =PUNGETC_MAX ; -memmove (buf, parsefile->nextc - unget, unget); - parsefile->nextc = buf += unget; +memmove (buf, parsefile->nextc - unget, unget + nr); + buf += unget; + parsefile->nextc = buf; + buf += nr; + nr =BUFSIZ - nr; +if (!IS_DEFINED_SMALL && !nr) +return nr; + + pnr = nr; retry: + nr = pnr;#ifndef SMALL if (parsefile->fd ==0 && el) {static const char *rl_cp;
@@ -216,9 +227,8 @@ retry:if (rl_cp ==NULL ) nr =0 ;else { - nr = el_len; -if (nr >BUFSIZ ) - nr =BUFSIZ ; +if (nr > el_len) + nr = el_len;memcpy (buf, rl_cp, nr);if (nr != el_len) { el_len -= nr;
@@ -230,10 +240,8 @@ retry: } else#endif if (parsefile->fd) - nr =read (parsefile->fd, buf,BUFSIZ ); + nr =read (parsefile->fd, buf, nr);else { -unsigned len =BUFSIZ ; - nr =0 ; do {
@@ -255,7 +263,7 @@ retry: } nr++; - }while (!IS_DEFINED_SMALL && *buf++ !='\n' && --len); + }while (0 ); }if (nr <0 ) {
@@ -290,19 +298,26 @@ static int preadbuffer(void)return PEOF ;flushall (); + q = parsefile->nextc; + something = !first; + more =input_get_lleft (parsefile);if (more <=0 ) { +int nr; + again: -if ((more =preadfd ()) <=0 ) { + nr = q - parsefile->nextc; + more =preadfd (); + q = parsefile->nextc + nr; +if (more <=0 ) {input_set_lleft (parsefile, parsefile->nleft =0 ); +if (!IS_DEFINED_SMALL && nr >0 ) + goto save;return PEOF ; } } - q = parsefile->nextc; -/* delete nul characters */ - something = !first;for (;;) {int c;
@@ -321,7 +336,6 @@ again:switch (c) {case '\n' : - parsefile->nleft = q - parsefile->nextc -1 ; goto done;default :
@@ -335,8 +349,7 @@ again: check:if (more <=0 ) { - parsefile->nleft = q - parsefile->nextc -1 ; -if (parsefile->nleft <0 ) +if (!IS_DEFINED_SMALL ) goto again;break ; }
@@ -344,6 +357,8 @@ check: done:input_set_lleft (parsefile, more); +save: + parsefile->nleft = q - parsefile->nextc -1 ;if (!IS_DEFINED_SMALL ) savec = *q; *q ='\0' ;
Signature · 1 line
2.39.2
PATCH v2 2/3 input: Use lseek on stdin when possible
In reply to Herbert XuFor files that can be sought, use lseek instead of reading one byte at a time.
--- src/eval.c | 3 ++ src/init.h | 1 + src/input.c | 108 +++++++++++++++++++++++++++++++++++--------------- src/input.h | 4 ++ src/jobs.c | 3 ++ src/mkinit.c | 6 +++ src/options.c | 2 +- src/redir.c | 26 +++++++----- src/trap.c | 1 + 9 files changed, 111 insertions(+), 43 deletions(-)
src/eval.c +3 -0
@@ -896,6 +896,8 @@ bail: goto bail;default : +flush_input (); +/* Fork off a child process if necessary. */ if (!(flags &EV_EXIT ) ||have_traps ()) {INTOFF ;
@@ -1131,6 +1133,7 @@ execcmd(int argc, char **argv) iflag =0 ;/* exit on error */ mflag =0 ;optschanged (); +flush_input ();shellexec (argv +1 ,pathval (),0 ); }return 0 ;
src/init.h +1 -0
@@ -39,4 +39,5 @@ union node;void init (void );void exitreset (void );void forkreset (union node *); +void postexitreset (void );void reset (void );
src/input.c +76 -32
196 lines not on this page · raw message
src/input.h +4 -0
@@ -35,6 +35,7 @@*/ #include <limits.h> +#include <stdbool.h> #ifdef SMALL #define IS_DEFINED_SMALL 1
@@ -94,6 +95,7 @@ struct parsefile { };extern struct parsefile *parsefile; +extern int stdin_istty;/* * The input line number. Input.c just defines this variable, and saves
@@ -113,6 +115,8 @@ void pushstdin(void);void popfile (void );void unwindfiles (struct parsefile *);void popallfiles (void ); +void flush_input (void ); +void reset_input (void );static inline int input_get_lleft (struct parsefile *pf) {
src/jobs.c +3 -0
@@ -968,6 +968,9 @@ forkshell(struct job *jp, union node *n, int mode)int pid;TRACE (("forkshell(%%%d, %p, %d) called\n" ,jobno (jp), n, mode)); + +flush_input (); + pid =fork ();if (pid ==0 )forkchild (jp, n, mode);
src/mkinit.c +6 -0
@@ -119,6 +119,11 @@ char forkreset[] = "\* This routine is called when we enter a subshell.\n\ */\n"; +char postexitreset[] = "\ +/*\n\ +* This routine is called in exitshell.\n\ +*/\n" ; +char reset[] ="\ /*\n\ * This routine is called when an error or an interrupt occurs in an\n\
@@ -130,6 +135,7 @@ struct event event[] = { {"INIT", "init", init}, {"EXITRESET", "exitreset", exitreset}, {"FORKRESET", "forkreset", forkreset, "union node *n"}, + {"POSTEXITRESET", "postexitreset", postexitreset}, {"RESET", "reset", reset}, {NULL, NULL} };
src/options.c +1 -1
@@ -142,7 +142,7 @@ procargs(int argc, char **argv)sh_error ("-c requires an argument" ); sflag =1 ; } -if (iflag ==2 && sflag ==1 &&isatty (0 ) &&isatty (1 )) +if (iflag ==2 && sflag ==1 && stdin_istty &&isatty (1 )) iflag =1 ;if (mflag ==2 ) mflag = iflag;
src/redir.c +16 -10
@@ -46,16 +46,17 @@* Code for dealing with input/output redirection. */ -#include "main.h" -#include "shell.h" -#include "nodes.h" -#include "jobs.h" -#include "options.h" -#include "expand.h" -#include "redir.h" -#include "output.h" -#include "memalloc.h" #include "error.h" +#include "expand.h" +#include "input.h" +#include "jobs.h" +#include "main.h" +#include "memalloc.h" +#include "nodes.h" +#include "options.h" +#include "output.h" +#include "redir.h" +#include "shell.h" #include "system.h" #include "trap.h"
@@ -142,6 +143,8 @@ redirect(union node *redir, int flags)continue ; fd = n->nfile.fd; +if (fd ==0 ) +reset_input ();if (sv) {int closed;
@@ -415,8 +418,11 @@ popredir(int drop)close (i);break ;default : -if (!drop) +if (!drop) { +if (i ==0 ) +reset_input ();dup2 (rp->renamed[i], i); + }close (rp->renamed[i]);break ; }
src/trap.c +1 -0
@@ -426,6 +426,7 @@ exitshell(void) } out:exitreset (); +postexitreset ();/* * Disable job control so that whoever had the foreground before we * started can get it back.
Signature · 1 line
2.39.2
PATCH v2 3/3 input: Use tee(2) for stdin pipe
In reply to Herbert Xuuse tee(2) to peek at pipes in order to avoid reading one byte at a time.
--- configure.ac | 2 +- src/input.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++------ src/system.h | 7 ++++++ 3 files changed, 72 insertions(+), 8 deletions(-)
configure.ac +1 -1
@@ -89,7 +89,7 @@ dnl Checks for library functions. AC_CHECK_FUNCS(bsearch faccessat getpwnam getrlimit isalpha killpg \ memfd_create mempcpy \ sigsetmask stpcpy strchrnul strsignal strtod strtoimax \ - strtoumax sysconf) + strtoumax sysconf tee) dnl Check whether it's worth working around FreeBSD PR kern/125009. dnl The traditional behavior of access/faccessat is crazy, but
src/input.c +64 -7
118 lines not on this page · raw message
src/system.h +7 -0
@@ -118,6 +118,13 @@ long sysconf(int) __attribute__((__noreturn__));int isblank (int c);#endif +#ifndef HAVE_TEE +static inline ssize_t tee (int fd_in,int fd_out,size_t len,unsigned int flags) +{ +return -1 ; +} +#endif +#ifndef HAVE_FNMATCH static inline int fnmatch (const char *pattern,const char *string,int flags) {
Signature · 1 line
2.39.2