Skip to contentSkip to conversation

rt-tests: Set affinity before applying numa

PATCH New John Kacur 1 participant 2 messages Sep 6, 2019

lore
Share & tools
lore

John Kacur

author
#0 lore raw

PATCH 1/2 rt-tests: Set affinity before applying numa

There are three changes here.

1. If affinity is not specified, but numa is available, then numa implies AFFINITY_USEALL.

2. Move setting affinity before applying numa
We need to set the affinity before applying numa, so that we don't apply numa settings to cpus we don't intend to run on.

3. Allow a greater number of threads than cpus to run in a round robin fashion in the case of numa.

---
 src/cyclictest/cyclictest.c | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)
src/cyclictest/cyclictest.c +21 -13
index d101e3d14854..52f93da7d074 100644
@@ -1349,8 +1349,12 @@ static void process_options (int argc, char *argv[], int max_cpus)
 	/* if smp wasn't requested, test for numa automatically */
 	if (!smp) {
 #ifdef NUMA
-		if (numa_available() != -1)
+		if (numa_available() != -1) {
 			numa = 1;
+			if (setaffinity == AFFINITY_UNSPECIFIED) {
+				setaffinity = AFFINITY_USEALL;
+			}
+		}
 #else
 		warn("cyclictest was not built with the numa option\n");
 		numa = 0;
@@ -1716,6 +1720,7 @@ int main(int argc, char **argv)
 	sigset_t sigset;
 	int signum = SIGALRM;
 	int mode;
+	int cpu;
 	int max_cpus = sysconf(_SC_NPROCESSORS_ONLN);
 	int i, ret = -1;
 	int status;
@@ -1877,6 +1882,16 @@ int main(int argc, char **argv)
 		if (status != 0)
 			fatal("error from pthread_attr_init for thread %d: %s\n", i, strerror(status));
 
+		switch (setaffinity) {
+		case AFFINITY_UNSPECIFIED: cpu = -1; break;
+		case AFFINITY_SPECIFIED:
+			cpu = cpu_for_thread(i, max_cpus);
+			if (verbose)
+				printf("Thread %d using cpu %d.\n", i, cpu);
+			break;
+		case AFFINITY_USEALL: cpu = i % max_cpus; break;
+		}
+
 		node = -1;
 		if (numa) {
 			void *stack;
@@ -1884,7 +1899,7 @@ int main(int argc, char **argv)
 			size_t stksize;
 
 			/* find the memory node associated with the cpu i */
-			node = rt_numa_numa_node_of_cpu(i);
+			node = rt_numa_numa_node_of_cpu(cpu);
 
 			/* get the stack size set for for this thread */
 			if (pthread_attr_getstack(&attr, &currstk, &stksize))
@@ -1895,7 +1910,7 @@ int main(int argc, char **argv)
 				stksize = PTHREAD_STACK_MIN * 2;
 
 			/*  allocate memory for a stack on appropriate node */
-			stack = rt_numa_numa_alloc_onnode(stksize, node, i);
+			stack = rt_numa_numa_alloc_onnode(stksize, node, cpu);
 
 			/* touch the stack pages to pre-fault them in */
 			memset(stack, 0, stksize);
@@ -1965,20 +1980,13 @@ int main(int argc, char **argv)
 			interval += distance;
 		if (verbose)
 			printf("Thread %d Interval: %d\n", i, interval);
+
 		par->max_cycles = max_cycles;
 		par->stats = stat;
 		par->node = node;
 		par->tnum = i;
-		switch (setaffinity) {
-		case AFFINITY_UNSPECIFIED: par->cpu = -1; break;
-		case AFFINITY_SPECIFIED:
-			par->cpu = cpu_for_thread(i, max_cpus);
-			if (verbose)
-				printf("Thread %d using cpu %d.\n", i,
-					par->cpu);
-			break;
-		case AFFINITY_USEALL: par->cpu = i % max_cpus; break;
-		}
+		par->cpu = cpu;
+
 		stat->min = 1000000;
 		stat->max = 0;
 		stat->avg = 0.0;
Signature · 1 line
2.20.1
author
#1 lore raw

PATCH 2/2 rt-tests: cyclictest: RFC - Get a snapshot of cyclictest without interuppting it

Right now if you are running cyclictest you can send it SIGUSR1 and it will spill out the data at a moment in time, and keep running.

However, this can be problematic if you are using another program such as rteval to consume the data.

This feature here lets you send SIGUSR2 to cyclictest and it dumps status to shared memory so you can read it without interrupting cyclictest and without interferring with the data it is outputing.

I haven't cleaned this up yet, but I thought it was kind of neat, so I wanted to share it with you a little before it is ready for prime time.

To use it, start cyclictest in one term, then in another term send it SIGUSR2, and then you can just cat the results like this:

    [jkacur@planxty rt-tests]$ ps ax | tail
    14083 ?        I      0:00 [kworker/1:0-events]
    14108 ?        I      0:01 [kworker/u16:3-events_unbound]
    14150 ?        I      0:00 [kworker/7:0-mm_percpu_wq]
    14287 pts/9    SLl+   0:14 ./cyclictest -t
    14324 ?        I      0:00 [kworker/1:2]
    14341 ?        I      0:00 [kworker/0:2]
    14342 ?        I      0:00 [kworker/2:1]
    14363 ?        I      0:00 [kworker/u16:0-events_unbound]
    14381 pts/2    R+     0:00 ps ax
    14382 pts/2    S+     0:00 tail

    [jkacur@planxty rt-tests]$ cat /dev/shm/cyclictest_shm
    [jkacur@planxty rt-tests]$ kill -s USR2 14287
    [jkacur@planxty rt-tests]$ cat /dev/shm/cyclictest_shm
    #---------------------------
    # cyclictest current status:
    T: 0 (14288) P: 0 I:1000 C: 124852 Min:     20 Act:   65 Avg:  104 Max:     354
    T: 1 (14289) P: 0 I:1500 C:  83235 Min:      6 Act:   77 Avg:   82 Max:     330
    T: 2 (14290) P: 0 I:2000 C:  62426 Min:      9 Act:   71 Avg:  110 Max:     358
    T: 3 (14291) P: 0 I:2500 C:  49940 Min:      9 Act:   77 Avg:  111 Max:     453
    T: 4 (14292) P: 0 I:3000 C:  41617 Min:     14 Act:   68 Avg:   69 Max:     815
    T: 5 (14293) P: 0 I:3500 C:  35672 Min:     15 Act:   68 Avg:  105 Max:     360
    T: 6 (14294) P: 0 I:4000 C:  31212 Min:     15 Act:   64 Avg:  103 Max:     372
    T: 7 (14295) P: 0 I:4500 C:  27744 Min:     20 Act:  118 Avg:   93 Max:     681
    #---------------------------
---
 src/cyclictest/cyclictest.c | 141 ++++++++++++++++++++++++++++++++++++
 1 file changed, 141 insertions(+)
src/cyclictest/cyclictest.c +141 -0
index 52f93da7d074..4d3fc25347b0 100644
@@ -230,10 +230,13 @@ static struct thread_param **parameters;
 static struct thread_stat **statistics;
 
 static void print_stat(FILE *fp, struct thread_param *par, int index, int verbose, int quiet);
+static void rstat_print_stat(struct thread_param *par, int index, int verbose, int quiet);
 
 static int latency_target_fd = -1;
 static int32_t latency_target_value = 0;
 
+static int rstat_fd = -1;
+
 /* Latency trick
  * if the file /dev/cpu_dma_latency exists,
  * open it and write a zero into it. This will tell
@@ -1461,6 +1464,22 @@ static void sighand(int sig)
 		fprintf(stderr, "#---------------------------\n");
 		quiet = oldquiet;
 		return;
+	} else if (sig == SIGUSR2) {
+		int i;
+		int oldquiet = quiet;
+
+		quiet = 0;
+		if (rstat_fd == -1) {
+			fprintf(stderr, "ERROR: rstat_fd not valid\n");
+			return;
+		}
+		dprintf(rstat_fd, "#---------------------------\n");
+		dprintf(rstat_fd, "# cyclictest current status:\n");
+		for (i = 0; i < num_threads; i++)
+			rstat_print_stat(parameters[i], i, 0, 0);
+		dprintf(rstat_fd, "#---------------------------\n");
+		quiet = oldquiet;
+		return;
 	}
 	shutdown = 1;
 	if (refresh_on_max)
@@ -1628,6 +1647,62 @@ static void print_stat(FILE *fp, struct thread_param *par, int index, int verbos
 	}
 }
 
+static void rstat_print_stat(struct thread_param *par, int index, int verbose, int quiet)
+{
+	struct thread_stat *stat = par->stats;
+	int fd = rstat_fd;
+
+	if (!verbose) {
+		if (quiet != 1) {
+			char *fmt;
+			if (use_nsecs)
+				fmt = "T:%2d (%5d) P:%2d I:%ld C:%7lu "
+				        "Min:%7ld Act:%8ld Avg:%8ld Max:%8ld";
+			else
+				fmt = "T:%2d (%5d) P:%2d I:%ld C:%7lu "
+				        "Min:%7ld Act:%5ld Avg:%5ld Max:%8ld";
+
+			dprintf(fd, fmt, index, stat->tid, par->prio,
+				par->interval, stat->cycles, stat->min,
+				stat->act, stat->cycles ?
+				(long)(stat->avg/stat->cycles) : 0, stat->max);
+
+			if (smi)
+				dprintf(fd," SMI:%8ld", stat->smi_count);
+
+			dprintf(fd, "\n");
+		}
+	} else {
+		while (stat->cycles != stat->cyclesread) {
+			unsigned long diff_smi;
+			long diff = stat->values
+			    [stat->cyclesread & par->bufmsk];
+
+			if (smi)
+				diff_smi = stat->smis
+				[stat->cyclesread & par->bufmsk];
+
+			if (diff > stat->redmax) {
+				stat->redmax = diff;
+				stat->cycleofmax = stat->cyclesread;
+			}
+			if (++stat->reduce == oscope_reduction) {
+				if (!smi)
+					dprintf(fd, "%8d:%8lu:%8ld\n", index,
+						stat->cycleofmax, stat->redmax);
+				else
+					dprintf(fd, "%8d:%8lu:%8ld%8ld\n",
+						index, stat->cycleofmax,
+						stat->redmax, diff_smi);
+
+				stat->reduce = 0;
+				stat->redmax = 0;
+			}
+			stat->cyclesread++;
+		}
+	}
+}
+
 
 /*
  * thread that creates a named fifo and hands out run stats when someone
@@ -1715,6 +1790,63 @@ static void trigger_update(struct thread_param *par, int diff, int64_t ts)
 	pthread_mutex_unlock(&trigger_lock);
 }
 
+/* Running status shared memory open */
+static int rstat_shm_open(void)
+{
+	int fd;
+	shm_unlink("/cyclictest_shm");
+
+	errno = 0;
+	fd = shm_open("/cyclictest_shm", O_RDWR|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH);
+
+	if (fd == -1) {
+		fprintf(stderr, "ERROR: shmopen %s\n", strerror(errno));
+	}
+
+	rstat_fd = fd;
+
+	return fd;
+}
+
+static int rstat_ftruncate(int fd)
+{
+	int err;
+
+	errno = 0;
+	err = ftruncate(fd, _SC_PAGE_SIZE);
+	if (err) {
+		fprintf(stderr, "ftruncate error %s\n", strerror(errno));
+	}
+
+	return err;
+}
+
+static void *rstat_mmap(int fd)
+{
+	void *mptr;
+
+	errno = 0;
+	mptr = mmap(0, _SC_PAGE_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+
+	if (mptr == (void*)-1) {
+		fprintf(stderr, "ERROR: mmap, %s\n", strerror(errno));
+	}
+
+	return mptr;
+}
+
+static void rstat_mlock(void *mptr)
+{
+	int err;
+
+	err = mlock(mptr, _SC_PAGE_SIZE);
+
+	errno = 0;
+	if (err == -1) {
+		fprintf(stderr, "ERROR, mlock %s\n", strerror(errno));
+	}
+}
+
 int main(int argc, char **argv)
 {
 	sigset_t sigset;
@@ -1855,6 +1987,7 @@ int main(int argc, char **argv)
 
 	}
 
+
 	mode = use_nanosleep + use_system;
 
 	sigemptyset(&sigset);
@@ -1864,6 +1997,14 @@ int main(int argc, char **argv)
 	signal(SIGINT, sighand);
 	signal(SIGTERM, sighand);
 	signal(SIGUSR1, sighand);
+	signal(SIGUSR2, sighand);
+
+	/* Set-up shm */
+	int sfd = rstat_shm_open();
+	void *mptr;
+	rstat_ftruncate(sfd);
+	mptr = rstat_mmap(sfd);
+	rstat_mlock(mptr);
 
 	parameters = calloc(num_threads, sizeof(struct thread_param *));
 	if (!parameters)
Signature · 1 line
2.20.1