Skip to contentSkip to conversation

Locally distributed work API

PATCH In review Richard Palethorpe via ltp 2 participants 9 messages Sep 27, 2022 – Sep 30, 2022 Library code

lore
Share & tools
lore
#0 lore raw

cover Locally distributed work API

Hello,

Sorry for such a large complicated patch-set. This is essentially an RFC, but I haven't marked it as such because some parts of it may be ready to merge.

This is motivated by patterns I noticed both in the read_all test and work we have been doing on the LTP parallel test executor (LTX). Currently LTX exists outside the main LTP tree and doesn't use the LTP library. However it would be nice to share code between some types of test and LTX.

We have various ways of communicating and synchronising between processes. This adds another which is particularly suited to distributing work across multiple processes. Including work that completes unevenly and is likely to fail.

Currently these are local processes, but LTX runs as a remote process on a SUT. It takes commands from and returns results to a test schedular on another machine. The interface should be relatively easy to extend to support this scenario.

Still TODO:
* Documenation (other than commit messages)
* Integrate LTX into LTP API
* Implement variations on the read_all test (e.g. splice_all)

Richard Palethorpe (6):
api/epoll: Add safe epoll functions
api/evloop: Add helpers for creating an event loop api/state_machine: Add validating state machines api/channel: Add channel abstraction for message passing api/worker: Add library for distributing work over multiple procs
read_all: Migrate to the worker lib

 include/tst_channel.h                   |  97 ++++
 include/tst_epoll.h                     |  36 ++
 include/tst_evloop.h                    |  32 ++
 include/tst_state_machine.h             |  50 ++
 include/tst_worker.h                    |  62 +++
 lib/tst_channel.c                       | 410 +++++++++++++++++
 lib/tst_epoll.c                         |  81 ++++
 lib/tst_evloop.c                        | 102 ++++
 lib/tst_state_machine.c                 |  98 ++++
 lib/tst_worker.c                        | 285 ++++++++++++
 testcases/kernel/fs/read_all/read_all.c | 588 +++++++-----------------
 11 files changed, 1424 insertions(+), 417 deletions(-)
 create mode 100644 include/tst_channel.h
 create mode 100644 include/tst_epoll.h
 create mode 100644 include/tst_evloop.h
 create mode 100644 include/tst_state_machine.h
 create mode 100644 include/tst_worker.h
 create mode 100644 lib/tst_channel.c
 create mode 100644 lib/tst_epoll.c
 create mode 100644 lib/tst_evloop.c
 create mode 100644 lib/tst_state_machine.c
 create mode 100644 lib/tst_worker.c
Signature · 5 lines
2.36.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#1 lore raw

PATCH 1/6 api/epoll: Add safe epoll functions

Probably safe to use it over (p)select/(p)poll now.

---
 include/tst_epoll.h | 36 ++++++++++++++++++++
 lib/tst_epoll.c     | 81 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+)
 create mode 100644 include/tst_epoll.h
 create mode 100644 lib/tst_epoll.c
include/tst_epoll.h new file +36 -0
new file mode 100644
index 000000000..c5ffc07e3
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2022 SUSE LLC <[email protected]>
+ */
+
+#include <sys/epoll.h>
+
+#ifndef TST_EPOLL_H
+#define TST_EPOLL_H
+
+typedef int (*tst_on_epoll_fn)(void *, uint32_t);
+struct tst_epoll_event_data {
+	tst_on_epoll_fn on_epoll;
+	void *self;
+};
+
+int safe_epoll_create1(const char *const file, const int lineno,
+		       const int flags);
+
+#define SAFE_EPOLL_CREATE1(flags) \
+	safe_epoll_create1(__FILE__, __LINE__, (flags))
+
+int safe_epoll_ctl(const char *const file, const int lineno,
+		   int epfd, int op, int fd, struct epoll_event *ev);
+
+#define SAFE_EPOLL_CTL(epfd, op, fd, ev) \
+	safe_epoll_ctl(__FILE__, __LINE__, epfd, op, fd, ev)
+
+int safe_epoll_wait(const char *const file, const int lineno,
+		    int epfd, struct epoll_event *events,
+		    int maxevents, int timeout);
+
+#define SAFE_EPOLL_WAIT(epfd, events, maxevents, timeout)\
+	safe_epoll_wait(__FILE__, __LINE__, epfd, events, maxevents, timeout)
+
+#endif
lib/tst_epoll.c new file +81 -0

81 lines not on this page · raw message

Signature · 5 lines
2.36.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#2 lore raw

PATCH 2/6 api/evloop: Add helpers for creating an event loop

In reply to Richard Palethorpe via ltp

Puts some of the boiler plate for creating an "event loop", into an API. Useful for asynchronous or evented I/O.

This uses epoll and signalfd which are very widely supported on Linux. I also think epoll is a better interface than ppoll and pselect.

The tst_epoll_event_data struct (added in the previous commit) can be used to add callbacks on particular FD events.

There is also a special callback for the signlfd and on_cont which is called at the end of each loop. Returning 0 from these will cause the loop to exit.

---
 include/tst_evloop.h |  32 ++++++++++++++
 lib/tst_evloop.c     | 102 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 include/tst_evloop.h
 create mode 100644 lib/tst_evloop.c
include/tst_evloop.h new file +32 -0
new file mode 100644
index 000000000..bdab2d6f7
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2022 SUSE LLC <[email protected]>
+ */
+
+#include "inttypes.h"
+#include "sys/signalfd.h"
+
+#include "tst_epoll.h"
+
+#ifndef TST_EVLOOP_H
+#define TST_EVLOOP_H
+
+struct tst_evloop {
+	int epollfd;
+	int signalfd;
+	struct tst_epoll_event_data signalfd_evdata;
+	int timeout;
+
+	void *priv;
+	int (*on_cont)(struct tst_evloop *self);
+	int (*on_signal)(struct tst_evloop *self, struct signalfd_siginfo *si);
+};
+
+void tst_evloop_setup(struct tst_evloop *self);
+void tst_evloop_run(struct tst_evloop *self);
+void tst_evloop_add(struct tst_evloop *self,
+		    struct tst_epoll_event_data *evdata,
+		    int fd, uint32_t events);
+void tst_evloop_cleanup(struct tst_evloop *self);
+
+#endif
lib/tst_evloop.c new file +102 -0

102 lines not on this page · raw message

Signature · 5 lines
2.36.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#3 lore raw

Re: [LTP] [PATCH 2/6] api/evloop: Add helpers for creating an event loop

Hi Richie,

Quoted 1 line

+++ b/lib/tst_evloop.c

...

lib/tst_evloop.c line 80

Quoted 11 lines
lib/tst_evloop.c new file +11 -0
+void tst_evloop_run(struct tst_evloop *self)
+{
+	static int saturated_warn;
+	const int maxevents = 128;
+	struct epoll_event events[maxevents];
+
+	for (;;) {
+		const int ev_num = SAFE_EPOLL_WAIT(self->epollfd, events,
+						   maxevents, self->timeout);
+
+		for (int i = 0; i < ev_num; i++)

I'm sorry we still do not allow to use C99 due CentOS 7 still supported:
lib/tst_evloop.c:80:3: error: 'for' loop initial declarations are only allowed in C99 mode

Kind regards,
Petr

Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
#4 lore raw

PATCH 3/6 api/state_machine: Add validating state machines

In reply to Richard Palethorpe via ltp

Allows creating state machines where the state transitions are validated. Also one can assert which states a line of code expects to be executed in.

This is useful for verifying implicit or explicit state machines used to process I/O events or data.

When a state violation is found a trace of previous state transitions is printed.

---
 include/tst_state_machine.h | 50 +++++++++++++++++++
 lib/tst_state_machine.c     | 98 +++++++++++++++++++++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100644 include/tst_state_machine.h
 create mode 100644 lib/tst_state_machine.c
include/tst_state_machine.h new file +50 -0
new file mode 100644
index 000000000..2e86535c6
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2022 SUSE LLC <[email protected]>
+ */
+
+#include "inttypes.h"
+
+#ifndef TST_STATE_MACHINE_H
+#define TST_STATE_MACHINE_H
+
+#define TST_STATE_ANY (~(uint64_t)0)
+
+struct tst_state_matrix {
+	char *names[64];
+	uint64_t states[64];
+};
+
+struct tst_state_trace {
+	const char *file;
+	int line;
+	unsigned from;
+	unsigned to;
+};
+
+struct tst_state_mach {
+	const struct tst_state_matrix *mat;
+
+	unsigned top;
+	struct tst_state_trace ring[8];
+};
+
+#define TST_STATE_SET(mach, to) \
+	tst_state_set(__FILE__, __LINE__, mach, to)
+
+void tst_state_set(const char *const file, const int lineno,
+		  struct tst_state_mach *mach, unsigned to);
+
+#define TST_STATE_EXP(mach, mask) \
+	tst_state_exp(__FILE__, __LINE__, mach, mask)
+
+void tst_state_exp(const char *const file, const int lineno,
+		   struct tst_state_mach *mach, uint64_t mask);
+
+#define TST_STATE_GET(mach, mask) \
+	tst_state_get(__FILE__, __LINE__, mach, mask)
+
+unsigned tst_state_get(const char *const file, const int lineno,
+		       struct tst_state_mach *mach, uint64_t mask);
+
+#endif
lib/tst_state_machine.c new file +98 -0

98 lines not on this page · raw message

Signature · 5 lines
2.36.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#5 lore raw

PATCH 4/6 api/channel: Add channel abstraction for message passing

In reply to Richard Palethorpe via ltp

Adds an API for the channel pattern and an implementation based on pipes.

A channel supports synchronous and asynchronous modes. When a message is sent synchronously it's guaranteed to have been sent and ack'ed by the time send returns. Similar for receiving.

In asynchronous mode, send returns immediately and a callback must be registered to handle the response. This requires that the channel is registered with an event loop. Again it's similar for receiving.

Messages are binary safe strings with a specified size.

The use of pipes instead of shared memory is motivated by the idea that the same interface can be used for remote or isolated processes.

---
 include/tst_channel.h |  97 ++++++++++
 lib/tst_channel.c     | 410 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 507 insertions(+)
 create mode 100644 include/tst_channel.h
 create mode 100644 lib/tst_channel.c
include/tst_channel.h new file +97 -0
new file mode 100644
index 000000000..c572d213b
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2022 SUSE LLC <[email protected]>
+ */
+
+#include "tst_evloop.h"
+#include "tst_state_machine.h"
+
+#ifndef TST_CHANNEL_H
+#define TST_CHANNEL_H
+
+enum tst_chan_mode {
+	CHM_SYNC,
+	CHM_ASYNC,
+};
+
+enum tst_chan_state {
+	CHS_CLOSED,
+	CHS_READY,
+	CHS_RECV,
+	CHS_SEND
+};
+
+struct tst_chan_buf {
+	char *ptr;
+	size_t len;
+	size_t off;
+};
+
+struct tst_chan;
+struct tst_chan_ops {
+	void (*const close)(struct tst_chan *self);
+
+	void (*const send)(struct tst_chan *self);
+	void (*const recv)(struct tst_chan *self);
+
+	int (*const on_epoll)(struct tst_chan *self, uint32_t events);
+};
+
+struct tst_chan {
+	const struct tst_chan_ops *ops;
+	void *priv;
+
+	enum tst_chan_mode mode;
+	struct tst_state_mach mach;
+	long long last_seen;
+
+	struct tst_epoll_event_data evdata;
+
+	struct tst_chan_buf in;
+	struct tst_chan_buf out;
+
+	void *user_priv;
+	void (*on_send)(struct tst_chan *self, char *sent, size_t len);
+	void (*on_recv)(struct tst_chan *self, char *recv, size_t len);
+};
+
+enum tst_pchan_msg_kind {
+	PCMK_ACK = 1,
+	PCMK_DATA
+};
+
+struct tst_pchan_envelope {
+	unsigned int kind;
+	unsigned int len;
+} __attribute__((packed));
+
+enum tst_pchan_state {
+	PCS_IDLE,
+	PCS_RECV_DATA,
+	PCS_SEND_ACK,
+	PCS_SEND_DATA,
+	PCS_RECV_ACK,
+};
+
+struct tst_pchan {
+	int infd;
+	int outfd;
+
+	struct tst_pchan_envelope envelope;
+	struct tst_chan_buf envelope_buf;
+
+	unsigned int out_full:1;
+
+	struct tst_state_mach mach;
+};
+
+void tst_chan_send(struct tst_chan *self, char *msg, size_t len);
+void tst_chan_recv(struct tst_chan *self, char *msg, size_t len);
+void tst_chan_seen(struct tst_chan *self);
+long long tst_chan_elapsed(struct tst_chan *self);
+
+void tst_pchan_open(struct tst_chan *self, int infd, int outfd,
+		    struct tst_evloop *const evloop);
+void tst_pchan_close(struct tst_chan *self);
+
+#endif
lib/tst_channel.c new file +410 -0

410 lines not on this page · raw message

Signature · 5 lines
2.36.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#6 lore raw

Re: [LTP] [PATCH 4/6] api/channel: Add channel abstraction for message passing

Hi Richie,

...

Quoted 1 line
lib/tst_channel.c +0 -0

...

lib/tst_channel.c line 177

Quoted 7 lines
lib/tst_channel.c +7 -0
+static void pipe_chan_send(struct tst_chan *self)
+{
+	struct tst_pchan *priv = self->priv;
+	const enum tst_pchan_state state =
+		TST_STATE_GET(&priv->mach, 1 << PCS_IDLE | 1 << PCS_SEND_DATA | 1 << PCS_RECV_ACK);
+
+ 	TST_STATE_EXP(&self->mach, 1 << CHS_READY | 1 << CHS_SEND);

nit: git am complains:

Applying: api/channel: Add channel abstraction for message passing .git/rebase-apply/patch:293: space before tab in indent.
TST_STATE_EXP(&self->mach, 1 << CHS_READY | 1 << CHS_SEND);

Petr

Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
#7 lore raw

PATCH 5/6 api/worker: Add library for distributing work over multiple procs

In reply to Richard Palethorpe via ltp

Builds on the channel, state machine and event loop APIs. Allows one to create multiple processes (workers) and distribute work to them using messages.

The main test process can create, send and receive messages from the workers. The main process can communicate async or synchronously with each worker over the worker's channel.

Initially workers are internally synchronous and do not communicate directly with each other. However there is nothing preventing them from creating an event loop or using channels between one another.

Presently workers are expected to be local processes. However various types of isolation could be used. The main process just needs to share a channel with them (pipe) and send and receive signals with them.

This is motivated by suggested additions to the read_all test and LTX.

---
 include/tst_worker.h |  62 ++++++++++
 lib/tst_worker.c     | 285 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 347 insertions(+)
 create mode 100644 include/tst_worker.h
 create mode 100644 lib/tst_worker.c
include/tst_worker.h new file +62 -0
new file mode 100644
index 000000000..b5c51a66c
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2022 SUSE LLC <[email protected]>
+ */
+
+#include "tst_channel.h"
+
+#ifndef TST_WORKER_H
+#define TST_WORKER_H
+
+enum tst_worker_mode {
+	TST_WORKER_SYNC,
+	TST_WORKER_ASYNC
+};
+
+enum tst_worker_state {
+	WS_STOPPED,
+	WS_RUNNING,
+	WS_STOPPING,
+	WS_KILL_SENT,
+	WS_DIED
+};
+
+struct tst_workers;
+struct tst_worker {
+	int i;
+	pid_t pid;
+	struct tst_chan chan;
+	struct tst_pchan pipe_chan;
+	struct tst_workers *group;
+	enum tst_worker_mode mode;
+	struct tst_state_mach mach;
+
+	char display_buf[128];
+	char *name;
+
+	void *priv;
+	int (*run)(struct tst_worker *self);
+	void (*on_stopped)(struct tst_worker *self);
+	void (*on_died)(struct tst_worker *self);
+	void (*on_timeout)(struct tst_worker *self);
+	void (*on_sent)(struct tst_worker *self, char *sent, size_t len);
+	void (*on_recved)(struct tst_worker *self, char *recv, size_t len);
+};
+
+struct tst_workers {
+	long long timeout;
+	struct tst_evloop evloop;
+
+	long count;
+	struct tst_worker *vec;
+};
+
+void tst_workers_setup(struct tst_workers *self);
+void tst_workers_cleanup(struct tst_workers *self);
+
+void tst_worker_start(struct tst_worker *self);
+int tst_worker_ttl(struct tst_worker *self);
+void tst_worker_kill(struct tst_worker *self);
+char *tst_worker_idstr(struct tst_worker *self);
+
+#endif
lib/tst_worker.c new file +285 -0

285 lines not on this page · raw message

Signature · 5 lines
2.36.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#8 lore raw

PATCH 6/6 read_all: Migrate to the worker lib

In reply to Richard Palethorpe via ltp

Use the worker API added in the previous commit.

---
 testcases/kernel/fs/read_all/read_all.c | 588 +++++++-----------------
 1 file changed, 171 insertions(+), 417 deletions(-)
testcases/kernel/fs/read_all/read_all.c +171 -417
index 266678ea7..6d3ff4594 100644
@@ -27,9 +27,9 @@
  * overridden with the 'w' parameters.
  */
 #include <signal.h>
+#include <stdint.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <sys/wait.h>
 #include <fnmatch.h>
 #include <lapi/fnmatch.h>
 #include <stdlib.h>
@@ -40,7 +40,6 @@
 #include <unistd.h>
 #include <string.h>
 #include <limits.h>
-#include <semaphore.h>
 #include <ctype.h>
 #include <pwd.h>
 #include <grp.h>
@@ -49,28 +48,12 @@
 #include "tst_safe_clocks.h"
 #include "tst_test.h"
 #include "tst_timer.h"
+#include "tst_worker.h"
 
-#define QUEUE_SIZE 16384
 #define BUFFER_SIZE 1024
 #define MAX_PATH 4096
 #define MAX_DISPLAY 40
 
-struct queue {
-	sem_t sem;
-	int front;
-	int back;
-	char data[QUEUE_SIZE];
-	char popped[BUFFER_SIZE];
-};
-
-struct worker {
-	int i;
-	pid_t pid;
-	struct queue *q;
-	int last_seen;
-	unsigned int kill_sent:1;
-};
-
 enum dent_action {
 	DA_UNKNOWN,
 	DA_IGNORE,
@@ -78,19 +61,21 @@ enum dent_action {
 	DA_VISIT,
 };
 
+struct path_worker {
+	char cur[MAX_PATH];
+	char next[MAX_PATH];
+};
+
 static char *verbose;
 static char *quiet;
 static char *root_dir;
 static char *str_reads;
 static int reads = 1;
 static char *str_worker_count;
-static long worker_count;
 static char *str_max_workers;
 static long max_workers = 15;
-static struct worker *workers;
 static char *drop_privs;
 static char *str_worker_timeout;
-static int worker_timeout;
 static int timeout_warnings_left = 15;
 
 static char *blacklist[] = {
@@ -101,83 +86,9 @@ static char *blacklist[] = {
 	"/sys/*/cpu??*(?)/*",	/* cpu* entries with 2 or more digits */
 };
 
-static long long epoch;
-
-static int atomic_timestamp(void)
-{
-	struct timespec now;
-
-	SAFE_CLOCK_GETTIME(CLOCK_MONOTONIC_RAW, &now);
-
-	return tst_timespec_to_us(now) - epoch;
-}
-
-static int queue_pop(struct queue *q)
-{
-	int i = q->front, j = 0;
-
-	sem_wait(&q->sem);
-
-	if (!q->data[i])
-		return 0;
-
-	while (q->data[i]) {
-		q->popped[j] = q->data[i];
-
-		if (++j >= BUFFER_SIZE - 1)
-			tst_brk(TBROK, "Buffer is too small for path");
-
-		 i = (i + 1) % QUEUE_SIZE;
-	}
-
-	q->popped[j] = '\0';
-	tst_atomic_store((i + 1) % QUEUE_SIZE, &q->front);
-
-	return 1;
-}
-
-static int queue_push(struct queue *q, const char *buf)
-{
-	int i = q->back, j = 0;
-	int front = tst_atomic_load(&q->front);
-
-	do {
-		q->data[i] = buf[j];
-
-		i = (i + 1) % QUEUE_SIZE;
-
-		if (i == front)
-			return 0;
-
-	} while (buf[j++]);
-
-	q->back = i;
-	sem_post(&q->sem);
-
-	return 1;
-}
-
-static struct queue *queue_init(void)
-{
-	struct queue *q = SAFE_MMAP(NULL, sizeof(*q),
-				    PROT_READ | PROT_WRITE,
-				    MAP_SHARED | MAP_ANONYMOUS,
-				    0, 0);
-
-	sem_init(&q->sem, 1, 0);
-	q->front = 0;
-	q->back = 0;
-
-	return q;
-}
-
-static void queue_destroy(struct queue *q, int is_worker)
-{
-	if (is_worker)
-		sem_destroy(&q->sem);
-
-	SAFE_MUNMAP(q, sizeof(*q));
-}
+static struct tst_workers workers;
+static struct path_worker *worker_privs;
+struct tst_evloop evloop;
 
 static void sanitize_str(char *buf, ssize_t count)
 {
@@ -208,63 +119,45 @@ static int is_blacklisted(const char *path)
 	return 0;
 }
 
-static void worker_heartbeat(const int worker)
-{
-	tst_atomic_store(atomic_timestamp(), &workers[worker].last_seen);
-}
-
-static int worker_elapsed(const int worker)
-{
-	struct worker *const w = workers + worker;
-
-	return atomic_timestamp() - tst_atomic_load(&w->last_seen);
-}
-
-static int worker_ttl(const int worker)
-{
-	return MAX(0, worker_timeout - worker_elapsed(worker));
-}
-
-static void read_test(const int worker, const char *const path)
+static void read_test(struct tst_worker *self, char *path)
 {
 	char buf[BUFFER_SIZE];
 	int fd;
 	ssize_t count;
-	const pid_t pid = workers[worker].pid;
-	int elapsed;
+	long long elapsed;
 
 	if (is_blacklisted(path))
 		return;
 
 	if (verbose)
-		tst_res(TINFO, "Worker %d: %s(%s)", pid, __func__, path);
+		tst_res(TINFO, "%s: %s(%s)", tst_worker_idstr(self), __func__, path);
 
 	fd = open(path, O_RDONLY | O_NONBLOCK);
 	if (fd < 0) {
 		if (!quiet) {
-			tst_res(TINFO | TERRNO, "Worker %d (%d): open(%s)",
-				pid, worker, path);
+			tst_res(TINFO | TERRNO, "%s: open(%s)",
+			        tst_worker_idstr(self), path);
 		}
 		return;
 	}
 
-	worker_heartbeat(worker);
+	elapsed = tst_chan_elapsed(&self->chan);
 	count = read(fd, buf, sizeof(buf) - 1);
-	elapsed = worker_elapsed(worker);
+	elapsed = tst_chan_elapsed(&self->chan) - elapsed;
 
 	if (count > 0 && verbose) {
 		sanitize_str(buf, count);
 		tst_res(TINFO,
-			"Worker %d (%d): read(%s, buf) = %zi, buf = %s, elapsed = %dus",
-			pid, worker, path, count, buf, elapsed);
+			"%s: read(%s, buf) = %zi, buf = %s, elapsed = %llus",
+			tst_worker_idstr(self), path, count, buf, elapsed);
 	} else if (!count && verbose) {
 		tst_res(TINFO,
-			"Worker %d (%d): read(%s) = EOF, elapsed = %dus",
-			pid, worker, path, elapsed);
+			"%s: read(%s) = EOF, elapsed = %llus",
+			tst_worker_idstr(self), path, elapsed);
 	} else if (count < 0 && !quiet) {
 		tst_res(TINFO | TERRNO,
-			"Worker %d (%d): read(%s), elapsed = %dus",
-			pid, worker, path, elapsed);
+			"%s: read(%s), elapsed = %llus",
+			tst_worker_idstr(self), path, elapsed);
 	}
 
 	SAFE_CLOSE(fd);
@@ -294,254 +187,213 @@ static void maybe_drop_privs(void)
 		tst_brk(TBROK | TTERRNO, "Failed to use nobody uid");
 }
 
-static int worker_run(int worker)
+static void visit_dir(struct tst_worker *self, const char *path)
 {
+	DIR *dir;
+	struct dirent *dent;
+	struct stat dent_st;
+	char dent_path[MAX_PATH];
+	enum dent_action act;
+
+	dir = opendir(path);
+	if (!dir) {
+		tst_res(TINFO | TERRNO, "opendir(%s)", path);
+		return;
+	}
+
+	while (1) {
+		errno = 0;
+		dent = readdir(dir);
+		if (!dent && errno) {
+			tst_res(TINFO | TERRNO, "readdir(%s)", path);
+			break;
+		} else if (!dent) {
+			break;
+		}
+
+		if (!strcmp(dent->d_name, ".") ||
+		    !strcmp(dent->d_name, ".."))
+			continue;
+
+		if (dent->d_type == DT_DIR)
+			act = DA_VISIT;
+		else if (dent->d_type == DT_LNK)
+			act = DA_IGNORE;
+		else if (dent->d_type == DT_UNKNOWN)
+			act = DA_UNKNOWN;
+		else
+			act = DA_READ;
+
+		snprintf(dent_path, MAX_PATH,
+			 "%s/%s", path, dent->d_name);
+
+		if (act == DA_UNKNOWN) {
+			if (lstat(dent_path, &dent_st))
+				tst_res(TINFO | TERRNO, "lstat(%s)", path);
+			else if ((dent_st.st_mode & S_IFMT) == S_IFDIR)
+				act = DA_VISIT;
+			else if ((dent_st.st_mode & S_IFMT) == S_IFLNK)
+				act = DA_IGNORE;
+			else
+				act = DA_READ;
+		}
+
+		if (act == DA_VISIT)
+			visit_dir(self, dent_path);
+		else if (act == DA_READ)
+			tst_chan_send(&self->chan, dent_path, strlen(dent_path) + 1);
+	}
+
+	if (closedir(dir))
+		tst_res(TINFO | TERRNO, "closedir(%s)", path);
+}
+
+static int dir_worker_run(struct tst_worker *self)
+{
+	visit_dir(self, root_dir);
+
+	tst_res(TINFO, "Dir Worker %d (%d): fin.", self->pid, self->i);
+	tst_chan_send(&self->chan, "", 1);
+
+	return 0;
+}
+
+static int path_worker_run(struct tst_worker *self)
+{
+	char buf[BUFFER_SIZE];
 	struct sigaction term_sa = {
 		.sa_handler = SIG_IGN,
 		.sa_flags = 0,
 	};
-	struct worker *const self = workers + worker;
-	struct queue *q = self->q;
 
 	sigaction(SIGTTIN, &term_sa, NULL);
 	maybe_drop_privs();
-	self->pid = getpid();
-
-	if (!worker_ttl(self->i)) {
-		tst_brk(TBROK,
-			"Worker timeout is too short; restarts take >%dus",
-			worker_elapsed(self->i));
-	}
 
 	while (1) {
-		worker_heartbeat(worker);
+		tst_chan_recv(&self->chan, buf, PATH_MAX);
 
-		if (!queue_pop(q))
+		if (!buf[0])
 			break;
 
-		read_test(worker, q->popped);
+		read_test(self, buf);
 	}
 
-	queue_destroy(q, 1);
 	tst_flush();
 	return 0;
 }
 
-static void spawn_workers(void)
+static void path_worker_resend(struct tst_worker *const self)
 {
-	int i;
-	struct worker *wa = workers;
-
-	memset(workers, 0, worker_count * sizeof(*workers));
+	struct path_worker *pw = self->priv;
 
-	for (i = 0; i < worker_count; i++) {
-		wa[i].i = i;
-		wa[i].q = queue_init();
-		wa[i].last_seen = atomic_timestamp();
-		wa[i].pid = SAFE_FORK();
-		if (!wa[i].pid)
-			exit(worker_run(i));
-	}
+	tst_chan_send(&self->chan, pw->next, strlen(pw->next));
 }
 
-static void restart_worker(const int worker)
+static void do_next_path(struct tst_worker *path_worker)
 {
-	struct worker *const w = workers + worker;
-	int wstatus, ret, i, q_len;
+	size_t slen = 1;
+	struct tst_worker *const dir_worker = workers.vec;
+	struct path_worker *pw = path_worker->priv;
 
-	if (!w->kill_sent) {
-		SAFE_KILL(w->pid, SIGKILL);
-		w->kill_sent = 1;
-		worker_heartbeat(worker);
-	}
-
-	ret = waitpid(w->pid, &wstatus, WNOHANG);
+	pw->next[0] = '\0';
 
-	if (!ret) {
-		if (worker_ttl(worker) > 0)
-			return;
+	if (TST_STATE_GET(&dir_worker->mach, TST_STATE_ANY) != WS_RUNNING)
+		goto send;
 
-		if (!quiet || timeout_warnings_left) {
-			tst_res(TINFO,
-				"Worker %d (%d): Timeout waiting after kill",
-				w->pid, worker);
-		}
-	} else if (ret != w->pid) {
-		tst_brk(TBROK | TERRNO, "Worker %d (%d): waitpid = %d",
-			w->pid, worker, ret);
-	}
+	tst_chan_recv(&dir_worker->chan, pw->next, BUFFER_SIZE);
 
-	w->kill_sent = 0;
+	if (!pw->next[0])
+		TST_STATE_SET(&dir_worker->mach, WS_STOPPING);
 
-	if (!w->q->popped[0]) {
-		tst_brk(TBROK,
-			"Worker %d (%d): Timed out, but doesn't appear to be reading anything",
-			w->pid, worker);
-	}
+	slen = dir_worker->chan.in.len;
+send:
+	tst_chan_send(&path_worker->chan, pw->next, slen);
 
-	if (!quiet || timeout_warnings_left) {
-		tst_res(TINFO, "Worker %d (%d): Last popped '%s'",
-			w->pid, worker, w->q->popped);
-	}
+}
 
-	/* Make sure the queue length and semaphore match. Threre is a
-	 * race in qeue_pop where the semaphore can be decremented
-	 * then the worker killed before updating q->front
-	 */
-	q_len = 0;
-	i = w->q->front;
-	while (i != w->q->back) {
-		if (!w->q->data[i])
-			q_len++;
-
-		i = (i + 1) % QUEUE_SIZE;
-	}
+static void path_worker_sent(struct tst_worker *self, char *path, size_t len)
+{
+	struct path_worker *pw = self->priv;
 
-	ret = sem_destroy(&w->q->sem);
-	if (ret == -1)
-		tst_brk(TBROK | TERRNO, "sem_destroy");
-	ret = sem_init(&w->q->sem, 1, q_len);
-	if (ret == -1)
-		tst_brk(TBROK | TERRNO, "sem_init");
+	memcpy(pw->cur, path, len);
 
-	worker_heartbeat(worker);
-	w->pid = SAFE_FORK();
+	if (!path[0])
+		return;
 
-	if (!w->pid)
-		exit(worker_run(worker));
+	do_next_path(self);
 }
 
-static void check_timeout_warnings_limit(void)
+static int check_timeout_warnings_limit(void)
 {
 	if (!quiet)
-		return;
+		return 1;
 
 	timeout_warnings_left--;
 
 	if (timeout_warnings_left)
-		return;
+		return 1;
 
 	tst_res(TINFO,
 		"Silencing timeout warnings; consider increasing LTP_RUNTIME_MUL or removing -q");
-}
-
-static int try_push_work(const int worker, const char *buf)
-{
-	int ret = 0;
-	int elapsed;
-	struct worker *const w = workers + worker;
-
-	if (w->kill_sent) {
-		restart_worker(worker);
-		return 0;
-	}
-
-	ret = queue_push(w->q, buf);
-	if (ret)
-		return 1;
-
-	elapsed = worker_elapsed(worker);
-
-	if (elapsed > worker_timeout) {
-		if (!quiet || timeout_warnings_left) {
-			tst_res(TINFO,
-				"Worker %d (%d): Stuck for %dus, restarting it",
-				w->pid, worker, elapsed);
-			check_timeout_warnings_limit();
-		}
-		restart_worker(worker);
-	}
 
 	return 0;
 }
 
-static void push_work(const int worker, const char *buf)
+static void path_worker_died(struct tst_worker *self)
 {
-	int sleep_time = 1;
+	struct path_worker *pw = self->priv;
 
-	while (!try_push_work(worker, buf)) {
-		const int ttl = worker_ttl(worker);
-
-		sleep_time = MIN(2 * sleep_time, ttl);
-		usleep(sleep_time);
+	if (pw->cur[0] == '\0') {
+		tst_brk(TBROK,
+			"%s: Died, but doesn't appear to be reading anything",
+			tst_worker_idstr(self));
 	}
-}
-
-static void stop_workers(void)
-{
-	const char stop_code[1] = { '\0' };
-	int i;
 
-	if (!workers)
-		return;
+	if (check_timeout_warnings_limit())
+		tst_res(TINFO, "%s: Died; Last sent '%s'", tst_worker_idstr(self), pw->cur);
 
-	for (i = 0; i < worker_count; i++) {
-		if (workers[i].q)
-			push_work(i, stop_code);
-	}
+	tst_worker_start(self);
+	path_worker_resend(self);
 }
 
-static void destroy_workers(void)
+static void spawn_workers(void)
 {
 	int i;
+	long wcount = workers.count;
+	struct tst_worker *wa = workers.vec;
 
-	if (!workers)
-		return;
-
-	for (i = 0; i < worker_count; i++) {
-		if (workers[i].q) {
-			queue_destroy(workers[i].q, 0);
-			workers[i].q = 0;
-		}
-	}
-}
-
-static int sched_work(const int first_worker,
-		      const char *path, int repetitions)
-{
-	int i, j;
-	int min_ttl = worker_timeout, sleep_time = 1;
-	int pushed, workers_pushed = 0;
-
-	for (i = 0, j = first_worker; i < repetitions; j++) {
-		if (j >= worker_count)
-			j = 0;
-
-		if (j == first_worker && !workers_pushed) {
-			sleep_time = MIN(2 * sleep_time, min_ttl);
-			usleep(sleep_time);
-			min_ttl = worker_timeout;
-		}
+	wa[0].name = "Dir";
+	wa[0].run = dir_worker_run;
+	wa[0].mode = TST_WORKER_SYNC;
 
-		if (j == first_worker)
-			workers_pushed = 0;
+	tst_worker_start(wa);
 
-		pushed = try_push_work(j, path);
-		i += pushed;
-		workers_pushed += pushed;
+	for (i = 1; i < wcount; i++) {
+		wa[i].name = "Path";
+		wa[i].run = path_worker_run;
+		wa[i].on_sent = path_worker_sent;
+		wa[i].on_died = path_worker_died;
+		wa[i].mode = TST_WORKER_ASYNC;
+		wa[i].priv = worker_privs + i;
 
-		if (!pushed)
-			min_ttl = MIN(min_ttl, worker_ttl(j));
+		tst_worker_start(wa + i);
+		do_next_path(wa + i);
 	}
-
-	return j;
 }
 
 static void setup(void)
 {
-	struct timespec now;
-
 	if (tst_parse_int(str_reads, &reads, 1, INT_MAX))
 		tst_brk(TBROK,
 			"Invalid reads (-r) argument: '%s'", str_reads);
 
-	if (tst_parse_long(str_max_workers, &max_workers, 1, LONG_MAX)) {
+	if (tst_parse_long(str_max_workers, &max_workers, 2, LONG_MAX)) {
 		tst_brk(TBROK,
 			"Invalid max workers (-w) argument: '%s'",
 			str_max_workers);
 	}
 
-	if (tst_parse_long(str_worker_count, &worker_count, 1, LONG_MAX)) {
+	if (tst_parse_long(str_worker_count, &workers.count, 2, LONG_MAX)) {
 		tst_brk(TBROK,
 			"Invalid worker count (-W) argument: '%s'",
 			str_worker_count);
@@ -550,139 +402,41 @@ static void setup(void)
 	if (!root_dir)
 		tst_brk(TBROK, "The directory argument (-d) is required");
 
-	if (!worker_count)
-		worker_count = MIN(MAX(tst_ncpus() - 1, 1L), max_workers);
-	workers = SAFE_MALLOC(worker_count * sizeof(*workers));
+	if (!workers.count)
+		workers.count = MIN(MAX(tst_ncpus() - 1, 2L), max_workers);
 
-	if (tst_parse_int(str_worker_timeout, &worker_timeout, 1, INT_MAX)) {
+	workers.vec = SAFE_MALLOC(workers.count * sizeof(workers.vec[0]));
+	memset(workers.vec, 0, workers.count * sizeof(workers.vec[0]));
+	worker_privs = SAFE_MALLOC(workers.count * sizeof(struct path_worker));
+
+	if (tst_parse_int(str_worker_timeout, (int *)&workers.timeout, 1, INT_MAX)) {
 		tst_brk(TBROK,
 			"Invalid worker timeout (-t) argument: '%s'",
 			str_worker_count);
 	}
 
-	if (worker_timeout) {
-		tst_res(TINFO, "Worker timeout forcibly set to %dms",
-			worker_timeout);
+	if (workers.timeout) {
+		tst_res(TINFO, "Worker timeout forcibly set to %lldms",
+			workers.timeout);
 	} else {
-		worker_timeout = 10 * tst_remaining_runtime();
-		tst_res(TINFO, "Worker timeout set to 10%% of max_runtime: %dms",
-			worker_timeout);
+		workers.timeout = 10 * tst_remaining_runtime();
+		tst_res(TINFO, "Worker timeout set to 10%% of max_runtime: %lldms",
+			workers.timeout);
 	}
-	worker_timeout *= 1000;
+	workers.timeout *= 1000;
 
-	SAFE_CLOCK_GETTIME(CLOCK_MONOTONIC_RAW, &now);
-	epoch = tst_timespec_to_us(now);
-}
-
-static void reap_children(void)
-{
-	int status, bad_exit = 0;
-	pid_t pid;
-
-	for (;;) {
-		pid = wait(&status);
-
-		if (pid > 0) {
-			if (!WIFEXITED(status))
-				bad_exit = 1;
-
-			continue;
-		}
-
-		if (errno == ECHILD)
-			break;
-
-		if (errno == EINTR)
-			continue;
-
-		tst_brk(TBROK | TERRNO, "wait() failed");
-	}
-
-	if (!bad_exit)
-		return;
-
-	tst_res(TINFO,
-		"Zombie workers detected; consider increasing LTP_RUNTIME_MUL");
+	tst_workers_setup(&workers);
 }
 
 static void cleanup(void)
 {
-	stop_workers();
-	reap_children();
-	destroy_workers();
-	free(workers);
-}
-
-static void visit_dir(const char *path)
-{
-	DIR *dir;
-	struct dirent *dent;
-	struct stat dent_st;
-	char dent_path[MAX_PATH];
-	enum dent_action act;
-	int last_sched = 0;
-
-	dir = opendir(path);
-	if (!dir) {
-		tst_res(TINFO | TERRNO, "opendir(%s)", path);
-		return;
-	}
-
-	while (1) {
-		errno = 0;
-		dent = readdir(dir);
-		if (!dent && errno) {
-			tst_res(TINFO | TERRNO, "readdir(%s)", path);
-			break;
-		} else if (!dent) {
-			break;
-		}
-
-		if (!strcmp(dent->d_name, ".") ||
-		    !strcmp(dent->d_name, ".."))
-			continue;
-
-		if (dent->d_type == DT_DIR)
-			act = DA_VISIT;
-		else if (dent->d_type == DT_LNK)
-			act = DA_IGNORE;
-		else if (dent->d_type == DT_UNKNOWN)
-			act = DA_UNKNOWN;
-		else
-			act = DA_READ;
-
-		snprintf(dent_path, MAX_PATH,
-			 "%s/%s", path, dent->d_name);
-
-		if (act == DA_UNKNOWN) {
-			if (lstat(dent_path, &dent_st))
-				tst_res(TINFO | TERRNO, "lstat(%s)", path);
-			else if ((dent_st.st_mode & S_IFMT) == S_IFDIR)
-				act = DA_VISIT;
-			else if ((dent_st.st_mode & S_IFMT) == S_IFLNK)
-				act = DA_IGNORE;
-			else
-				act = DA_READ;
-		}
-
-		if (act == DA_VISIT)
-			visit_dir(dent_path);
-		else if (act == DA_READ)
-			last_sched = sched_work(last_sched, dent_path, reads);
-	}
-
-	if (closedir(dir))
-		tst_res(TINFO | TERRNO, "closedir(%s)", path);
+	tst_workers_cleanup(&workers);
 }
 
 static void run(void)
 {
 	spawn_workers();
-	visit_dir(root_dir);
-
-	stop_workers();
-	reap_children();
-	destroy_workers();
+	tst_evloop_run(&workers.evloop);
 
 	tst_res(TPASS, "Finished reading files");
 }
Signature · 5 lines
2.36.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp