gem_wsim: Rip out userspace balancing

71 lines not on this page · raw message

1,826 lines not on this page · raw message

104 lines not on this page · raw message

index 5cc71c56fe6e..dc47470c621d 100644
@@ -88,14 +88,21 @@ enum w_type
 	LOAD_BALANCE,
 	BOND,
 	TERMINATE,
-	SSEU
+	SSEU,
+	WORKINGSET,
+};
+
+struct dep_entry {
+	int target;
+	bool write;
+	int working_set; /* -1 = step dependecy, >= 0 working set id */
 };
 
 struct deps
 {
 	int nr;
 	bool submit_fence;
-	int *list;
+	struct dep_entry *list;
 };
 
 struct w_arg {

@@ -110,6 +117,14 @@ struct bond {
 	enum intel_engine_id master;
 };
 
+struct working_set {
+	int id;
+	bool shared;
+	unsigned int nr;
+	uint32_t *handles;
+	unsigned long *sizes;
+};
+
 struct workload;
 
 struct w_step

@@ -143,6 +158,7 @@ struct w_step
 			enum intel_engine_id bond_master;
 		};
 		int sseu;
+		struct working_set working_set;
 	};
 
 	/* Implementation details */

@@ -193,6 +209,9 @@ struct workload
 	unsigned int nr_ctxs;
 	struct ctx *ctx_list;
 
+	struct working_set **working_sets; /* array indexed by set id */
+	int max_working_set_id;
+
 	int sync_timeline;
 	uint32_t sync_seqno;
 

@@ -281,11 +300,129 @@ print_engine_calibrations(void)
 	printf("\n");
 }
 
+static void add_dep(struct deps *deps, struct dep_entry entry)
+{
+	deps->list = realloc(deps->list, sizeof(*deps->list) * (deps->nr + 1));
+	igt_assert(deps->list);
+
+	deps->list[deps->nr++] = entry;
+}
+
+static int
+parse_working_set_deps(struct workload *wrk,
+		       struct deps *deps,
+		       struct dep_entry _entry,
+		       char *str)
+{
+	/*
+	 * 1 - target handle index in the specified working set.
+	 * 2-4 - range
+	 */
+	struct dep_entry entry = _entry;
+	char *s;
+
+	s = index(str, '-');
+	if (s) {
+		int from, to;
+
+		from = atoi(str);
+		if (from < 0)
+			return -1;
+
+		to = atoi(++s);
+		if (to <= 0)
+			return -1;
+
+		if (to <= from)
+			return -1;
+
+		for (entry.target = from; entry.target <= to; entry.target++)
+			add_dep(deps, entry);
+	} else {
+		entry.target = atoi(str);
+		if (entry.target < 0)
+			return -1;
+
+		add_dep(deps, entry);
+	}
+
+	return 0;
+}
+
+static int
+parse_dependency(unsigned int nr_steps, struct w_step *w, char *str)
+{
+	struct dep_entry entry = { .working_set = -1 };
+	bool submit_fence = false;
+	char *s;
+
+	switch (str[0]) {
+	case '-':
+		if (str[1] < '0' || str[1] > '9')
+			return -1;
+
+		entry.target = atoi(str);
+		if (entry.target > 0 || ((int)nr_steps + entry.target) < 0)
+			return -1;
+
+		add_dep(&w->data_deps, entry);
+
+		break;
+	case 's':
+		submit_fence = true;
+		/* Fall-through. */
+	case 'f':
+		/* Multiple fences not yet supported. */
+		igt_assert_eq(w->fence_deps.nr, 0);
+
+		entry.target = atoi(++str);
+		if (entry.target > 0 || ((int)nr_steps + entry.target) < 0)
+			return -1;
+
+		add_dep(&w->fence_deps, entry);
+
+		w->fence_deps.submit_fence = submit_fence;
+		break;
+	case 'w':
+		entry.write = true;
+		/* Fall-through. */
+	case 'r':
+		/*
+		 * [rw]N-<str>
+		 * r1-<str> or w2-<str>, where N is working set id.
+		 */
+		s = index(++str, '-');
+		if (!s)
+			return -1;
+
+		entry.working_set = atoi(str);
+		if (entry.working_set < 0)
+			return -1;
+
+		if (parse_working_set_deps(w->wrk, &w->data_deps, entry, ++s))
+			return -1;
+
+		break;
+	default:
+		return -1;
+	};
+
+	return 0;
+}
+
 static int
 parse_dependencies(unsigned int nr_steps, struct w_step *w, char *_desc)
 {
 	char *desc = strdup(_desc);
 	char *token, *tctx = NULL, *tstart = desc;
+	int ret = 0;
+
+	/*
+	 * Skip when no dependencies to avoid having to detect
+	 * non-sensical "0/0/..." below.
+	 */
+	if (!strcmp(_desc, "0"))
+		goto out;
 
 	igt_assert(desc);
 	igt_assert(!w->data_deps.nr && w->data_deps.nr == w->fence_deps.nr);

@@ -293,47 +430,17 @@ parse_dependencies(unsigned int nr_steps, struct w_step *w, char *_desc)
 		   w->data_deps.list == w->fence_deps.list);
 
 	while ((token = strtok_r(tstart, "/", &tctx)) != NULL) {
-		bool submit_fence = false;
-		char *str = token;
-		struct deps *deps;
-		int dep;
-
 		tstart = NULL;
 
-		if (str[0] == '-' || (str[0] >= '0' && str[0] <= '9')) {
-			deps = &w->data_deps;
-		} else {
-			if (str[0] == 's')
-				submit_fence = true;
-			else if (str[0] != 'f')
-				return -1;
-
-			deps = &w->fence_deps;
-			str++;
-		}
-
-		dep = atoi(str);
-		if (dep > 0 || ((int)nr_steps + dep) < 0) {
-			if (deps->list)
-				free(deps->list);
-			return -1;
-		}
-
-		if (dep < 0) {
-			deps->nr++;
-			/* Multiple fences not yet supported. */
-			igt_assert(deps->nr == 1 || deps != &w->fence_deps);
-			deps->list = realloc(deps->list,
-					     sizeof(*deps->list) * deps->nr);
-			igt_assert(deps->list);
-			deps->list[deps->nr - 1] = dep;
-			deps->submit_fence = submit_fence;
-		}
+		ret = parse_dependency(nr_steps, w, token);
+		if (ret)
+			break;
 	}
 
+out:
 	free(desc);
 
-	return 0;
+	return ret;
 }
 
 static void __attribute__((format(printf, 1, 2)))

@@ -624,6 +731,101 @@ static int parse_engine_map(struct w_step *step, const char *_str)
 	return 0;
 }
 
+static unsigned long parse_size(char *str)
+{
+	const unsigned int len = strlen(str);
+	unsigned int mult = 1;
+	long val;
+
+	/* "1234567890[gGmMkK]" */
+
+	if (len == 0)
+		return 0;
+
+	switch (str[len - 1]) {
+	case 'g':
+	case 'G':
+		mult *= 1024;
+		/* Fall-throuogh. */
+	case 'm':
+	case 'M':
+		mult *= 1024;
+		/* Fall-throuogh. */
+	case 'k':
+	case 'K':
+		mult *= 1024;
+
+		str[len - 1] = 0;
+		/* Fall-throuogh. */
+
+	case '0' ... '9':
+		break;
+	default:
+		return 0; /* Unrecognized non-digit. */
+	}
+
+	val = atol(str);
+	if (val <= 0)
+		return 0;
+
+	return val * mult;
+}
+
+static int add_buffers(struct working_set *set, char *str)
+{
+	/*
+	 * 4096
+	 * 4k
+	 * 4m
+	 * 4g
+	 * 10n4k - 10 4k batches
+	 */
+	unsigned long *sizes, size;
+	int add, i;
+	char *n;
+
+	n = index(str, 'n');
+	if (n) {
+		*n = 0;
+		add = atoi(str);
+		if (add <= 0)
+			return -1;
+		str = ++n;
+	} else {
+		add = 1;
+	}
+
+	size = parse_size(str);
+	if (!size)
+		return -1;
+
+	sizes = realloc(set->sizes, (set->nr + add) * sizeof(*sizes));
+	if (!sizes)
+		return -1;
+
+	for (i = 0; i < add; i++)
+		sizes[set->nr + i] = size;
+
+	set->nr += add;
+	set->sizes = sizes;
+
+	return 0;
+}
+
+static int parse_working_set(struct working_set *set, char *str)
+{
+	char *token, *tctx = NULL, *tstart = str;
+
+	while ((token = strtok_r(tstart, "/", &tctx))) {
+		tstart = NULL;
+
+		if (add_buffers(set, token))
+			return -1;
+	}
+
+	return 0;
+}
+
 static uint64_t engine_list_mask(const char *_str)
 {
 	uint64_t mask = 0;

@@ -644,6 +846,8 @@ static uint64_t engine_list_mask(const char *_str)
 	return mask;
 }
 
+static void allocate_working_set(struct working_set *set);
+
 #define int_field(_STEP_, _FIELD_, _COND_, _ERR_) \
 	if ((field = strtok_r(fstart, ".", &fctx))) { \
 		tmp = atoi(field); \

@@ -661,7 +865,7 @@ parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
 	char *desc = strdup(arg->desc);
 	char *_token, *token, *tctx = NULL, *tstart = desc;
 	char *field, *fctx = NULL, *fstart;
-	struct w_step step, *steps = NULL;
+	struct w_step step, *w, *steps = NULL;
 	unsigned int valid;
 	int i, j, tmp;
 

@@ -851,6 +1055,28 @@ parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
 
 				step.type = BOND;
 				goto add_step;
+			} else if (!strcmp(field, "w") || !strcmp(field, "W")) {
+				unsigned int nr = 0;
+
+				step.working_set.shared = field[0] == 'W';
+
+				while ((field = strtok_r(fstart, ".", &fctx))) {
+					tmp = atoi(field);
+					if (nr == 0) {
+						step.working_set.id = tmp;
+					} else {
+						tmp = parse_working_set(&step.working_set,
+									field);
+						check_arg(tmp < 0,
+							  "Invalid working set at step %u!\n",
+							  nr_steps);
+					}
+
+					nr++;
+				}
+
+				step.type = WORKINGSET;
+				goto add_step;
 			}
 
 			if (!field) {

@@ -975,6 +1201,8 @@ add_step:
 	wrk->steps = steps;
 	wrk->prio = arg->prio;
 	wrk->sseu = arg->sseu;
+	wrk->max_working_set_id = -1;
+	wrk->working_sets = NULL;
 
 	free(desc);
 

@@ -984,7 +1212,7 @@ add_step:
 	 */
 	for (i = 0; i < nr_steps; i++) {
 		for (j = 0; j < steps[i].fence_deps.nr; j++) {
-			tmp = steps[i].idx + steps[i].fence_deps.list[j];
+			tmp = steps[i].idx + steps[i].fence_deps.list[j].target;
 			check_arg(tmp < 0 || tmp >= i ||
 				  (steps[tmp].type != BATCH &&
 				   steps[tmp].type != SW_FENCE),

@@ -1003,6 +1231,51 @@ add_step:
 		}
 	}
 
+	/*
+	 * Check no duplicate working set ids.
+	 */
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		struct w_step *w2;
+
+		if (w->type != WORKINGSET)
+			continue;
+
+		for (j = 0, w2 = wrk->steps; j < wrk->nr_steps; w2++, j++) {
+			if (j == i)
+				continue;
+			if (w2->type != WORKINGSET)
+				continue;
+
+			check_arg(w->working_set.id == w2->working_set.id,
+				  "Duplicate working set id at %u!\n", j);
+		}
+	}
+
+	/*
+	 * Allocate shared working sets.
+	 */
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		if (w->type == WORKINGSET && w->working_set.shared)
+			allocate_working_set(&w->working_set);
+	}
+
+	wrk->max_working_set_id = -1;
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		if (w->type == WORKINGSET &&
+		    w->working_set.shared &&
+		    w->working_set.id > wrk->max_working_set_id)
+			wrk->max_working_set_id = w->working_set.id;
+	}
+
+	wrk->working_sets = calloc(wrk->max_working_set_id + 1,
+				   sizeof(*wrk->working_sets));
+	igt_assert(wrk->working_sets);
+
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		if (w->type == WORKINGSET && w->working_set.shared)
+			wrk->working_sets[w->working_set.id] = &w->working_set;
+	}
+
 	return wrk;
 }
 

@@ -1024,6 +1297,18 @@ clone_workload(struct workload *_wrk)
 
 	memcpy(wrk->steps, _wrk->steps, sizeof(struct w_step) * wrk->nr_steps);
 
+	wrk->max_working_set_id = _wrk->max_working_set_id;
+	if (wrk->max_working_set_id >= 0) {
+		wrk->working_sets = calloc(wrk->max_working_set_id + 1,
+					sizeof(*wrk->working_sets));
+		igt_assert(wrk->working_sets);
+
+		memcpy(wrk->working_sets,
+		       _wrk->working_sets,
+		       (wrk->max_working_set_id + 1) *
+		       sizeof(*wrk->working_sets));
+	}
+
 	/* Check if we need a sw sync timeline. */
 	for (i = 0; i < wrk->nr_steps; i++) {
 		if (wrk->steps[i].type == SW_FENCE) {

@@ -1222,17 +1507,36 @@ alloc_step_batch(struct workload *wrk, struct w_step *w)
 	igt_assert(j < nr_obj);
 
 	for (i = 0; i < w->data_deps.nr; i++) {
-		igt_assert(w->data_deps.list[i] <= 0);
-		if (w->data_deps.list[i]) {
-			int dep_idx = w->idx + w->data_deps.list[i];
+		struct dep_entry *entry = &w->data_deps.list[i];
+		uint32_t dep_handle;
 
+		if (entry->working_set == -1) {
+			int dep_idx = w->idx + entry->target;
+
+			igt_assert(entry->target <= 0);
 			igt_assert(dep_idx >= 0 && dep_idx < w->idx);
 			igt_assert(wrk->steps[dep_idx].type == BATCH);
 
-			w->obj[j].handle = wrk->steps[dep_idx].obj[0].handle;
-			j++;
-			igt_assert(j < nr_obj);
+			dep_handle = wrk->steps[dep_idx].obj[0].handle;
+		} else {
+			struct working_set *set;
+
+			igt_assert(entry->working_set <=
+				   wrk->max_working_set_id);
+
+			set = wrk->working_sets[entry->working_set];
+
+			igt_assert(set->nr);
+			igt_assert(entry->target < set->nr);
+			igt_assert(set->sizes[entry->target]);
+
+			dep_handle = set->handles[entry->target];
 		}
+
+		w->obj[j].flags = entry->write ? EXEC_OBJECT_WRITE : 0;
+		w->obj[j].handle = dep_handle;
+		j++;
+		igt_assert(j < nr_obj);
 	}
 
 	if (w->unbound_duration)

@@ -1391,10 +1695,22 @@ static size_t sizeof_engines_bond(int count)
 			engines[count]);
 }
 
+static void allocate_working_set(struct working_set *set)
+{
+	unsigned int i;
+
+	set->handles = calloc(set->nr, sizeof(*set->handles));
+	igt_assert(set->handles);
+
+	for (i = 0; i < set->nr; i++)
+		set->handles[i] = gem_create(fd, set->sizes[i]);
+}
+
 #define alloca0(sz) ({ size_t sz__ = (sz); memset(alloca(sz__), 0, sz__); })
 
 static int prepare_workload(unsigned int id, struct workload *wrk)
 {
+	struct working_set **sets;
 	uint32_t share_vm = 0;
 	int max_ctx = -1;
 	struct w_step *w;

@@ -1629,6 +1945,51 @@ static int prepare_workload(unsigned int id, struct workload *wrk)
 		}
 	}
 
+	/*
+	 * Allocate working sets.
+	 */
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		if (w->type == WORKINGSET && !w->working_set.shared)
+			allocate_working_set(&w->working_set);
+	}
+
+	/*
+	 * Map of working set ids.
+	 */
+	wrk->max_working_set_id = -1;
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		if (w->type == WORKINGSET &&
+		    w->working_set.id > wrk->max_working_set_id)
+			wrk->max_working_set_id = w->working_set.id;
+	}
+
+	sets = wrk->working_sets;
+	wrk->working_sets = calloc(wrk->max_working_set_id + 1,
+				   sizeof(*wrk->working_sets));
+	igt_assert(wrk->working_sets);
+
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		struct working_set *set;
+
+		if (w->type != WORKINGSET)
+			continue;
+
+		if (!w->working_set.shared) {
+			set = &w->working_set;
+		} else {
+			igt_assert(sets);
+
+			set = sets[w->working_set.id];
+			igt_assert(set->shared);
+			igt_assert(set->sizes);
+		}
+
+		wrk->working_sets[w->working_set.id] = set;
+	}
+
+	if (sets)
+		free(sets);
+
 	/*
 	 * Allocate batch buffers.
 	 */

@@ -1698,7 +2059,7 @@ do_eb(struct workload *wrk, struct w_step *w, enum intel_engine_id engine)
 		      2 * sizeof(uint32_t));
 
 	for (i = 0; i < w->fence_deps.nr; i++) {
-		int tgt = w->idx + w->fence_deps.list[i];
+		int tgt = w->idx + w->fence_deps.list[i].target;
 
 		/* TODO: fence merging needed to support multiple inputs */
 		igt_assert(i == 0);

@@ -1729,14 +2090,18 @@ static void sync_deps(struct workload *wrk, struct w_step *w)
 	unsigned int i;
 
 	for (i = 0; i < w->data_deps.nr; i++) {
+		struct dep_entry *entry = &w->data_deps.list[i];
 		int dep_idx;
 
-		igt_assert(w->data_deps.list[i] <= 0);
+		if (entry->working_set == -1)
+			continue;
+
+		igt_assert(entry->target <= 0);
 
-		if (!w->data_deps.list[i])
+		if (!entry->target)
 			continue;
 
-		dep_idx = w->idx + w->data_deps.list[i];
+		dep_idx = w->idx + entry->target;
 
 		igt_assert(dep_idx >= 0 && dep_idx < w->idx);
 		igt_assert(wrk->steps[dep_idx].type == BATCH);

@@ -1836,11 +2201,6 @@ static void *run_workload(void *data)
 					MI_BATCH_BUFFER_END;
 				__sync_synchronize();
 				continue;
-			} else if (w->type == PREEMPTION ||
-				   w->type == ENGINE_MAP ||
-				   w->type == LOAD_BALANCE ||
-				   w->type == BOND) {
-				continue;
 			} else if (w->type == SSEU) {
 				if (w->sseu != wrk->ctx_list[w->context * 2].sseu) {
 					wrk->ctx_list[w->context * 2].sseu =

@@ -1848,6 +2208,13 @@ static void *run_workload(void *data)
 							     w->sseu);
 				}
 				continue;
+			} else if (w->type == PREEMPTION ||
+				   w->type == ENGINE_MAP ||
+				   w->type == LOAD_BALANCE ||
+				   w->type == BOND ||
+				   w->type == WORKINGSET) {
+				   /* No action for these at execution time. */
+				continue;
 			}
 
 			if (do_sleep || w->type == PERIOD) {

index dc47470c621d..dd0b2e260a5a 100644
@@ -2117,7 +2117,8 @@ static void *run_workload(void *data)
 	struct w_step *w;
 	int throttle = -1;
 	int qd_throttle = -1;
-	int count;
+	int count, missed = 0;
+	unsigned long time_tot = 0, time_min = ULONG_MAX, time_max = 0;
 	int i;
 
 	clock_gettime(CLOCK_MONOTONIC, &t_start);

@@ -2137,12 +2138,19 @@ static void *run_workload(void *data)
 				do_sleep = w->delay;
 			} else if (w->type == PERIOD) {
 				struct timespec now;
+				int elapsed;
 
 				clock_gettime(CLOCK_MONOTONIC, &now);
-				do_sleep = w->period -
-					   elapsed_us(&wrk->repeat_start, &now);
+				elapsed = elapsed_us(&wrk->repeat_start, &now);
+				do_sleep = w->period - elapsed;
+				time_tot += elapsed;
+				if (elapsed < time_min)
+					time_min = elapsed;
+				if (elapsed > time_max)
+					time_max = elapsed;
 				if (do_sleep < 0) {
-					if (verbose > 1)
+					missed++;
+					if (verbose > 2)
 						printf("%u: Dropped period @ %u/%u (%dus late)!\n",
 						       wrk->id, count, i, do_sleep);
 					continue;

@@ -2296,6 +2304,9 @@ static void *run_workload(void *data)
 		printf("%c%u: %.3fs elapsed (%d cycles, %.3f workloads/s).",
 		       wrk->background ? ' ' : '*', wrk->id,
 		       t, count, count / t);
+		if (time_tot)
+			printf(" Time avg/min/max=%lu/%lu/%luus; %u missed.",
+			       time_tot / count, time_min, time_max, missed);
 		putchar('\n');
 	}
 

index dd0b2e260a5a..23b81d181df3 100644
@@ -1490,6 +1490,11 @@ get_ctxid(struct workload *wrk, struct w_step *w)
 	return wrk->ctx_list[w->context].id;
 }
 
+static uint32_t alloc_bo(int i915, unsigned long size)
+{
+	return gem_create(i915, size);
+}
+
 static void
 alloc_step_batch(struct workload *wrk, struct w_step *w)
 {

@@ -1501,7 +1506,7 @@ alloc_step_batch(struct workload *wrk, struct w_step *w)
 	w->obj = calloc(nr_obj, sizeof(*w->obj));
 	igt_assert(w->obj);
 
-	w->obj[j].handle = gem_create(fd, 4096);
+	w->obj[j].handle = alloc_bo(fd, 4096);
 	w->obj[j].flags = EXEC_OBJECT_WRITE;
 	j++;
 	igt_assert(j < nr_obj);

@@ -1546,7 +1551,8 @@ alloc_step_batch(struct workload *wrk, struct w_step *w)
 	else
 		w->bb_sz = get_bb_sz(w, w->duration.max);
 
-	w->bb_handle = w->obj[j].handle = gem_create(fd, w->bb_sz + (w->unbound_duration ? 4096 : 0));
+	w->bb_handle = w->obj[j].handle =
+		alloc_bo(fd, w->bb_sz + (w->unbound_duration ? 4096 : 0));
 	init_bb(w);
 	w->obj[j].relocation_count = terminate_bb(w);
 

@@ -1703,7 +1709,7 @@ static void allocate_working_set(struct working_set *set)
 	igt_assert(set->handles);
 
 	for (i = 0; i < set->nr; i++)
-		set->handles[i] = gem_create(fd, set->sizes[i]);
+		set->handles[i] = alloc_bo(fd, set->sizes[i]);
 }
 
 #define alloca0(sz) ({ size_t sz__ = (sz); memset(alloca(sz__), 0, sz__); })

@@ -2339,7 +2345,7 @@ static unsigned long calibrate_nop(unsigned int tolerance_pct, struct intel_exec
 	do {
 		struct timespec t_start;
 
-		obj.handle = gem_create(fd, size);
+		obj.handle = alloc_bo(fd, size);
 		gem_write(fd, obj.handle, size - sizeof(bbe), &bbe,
 			  sizeof(bbe));
 		gem_execbuf(fd, &eb);

index 23b81d181df3..a387e180c242 100644
@@ -117,12 +117,18 @@ struct bond {
 	enum intel_engine_id master;
 };
 
+struct work_buffer_size {
+	unsigned long size;
+	unsigned long min;
+	unsigned long max;
+};
+
 struct working_set {
 	int id;
 	bool shared;
 	unsigned int nr;
 	uint32_t *handles;
-	unsigned long *sizes;
+	struct work_buffer_size *sizes;
 };
 
 struct workload;

@@ -203,6 +209,7 @@ struct workload
 	bool print_stats;
 
 	uint32_t bb_prng;
+	uint32_t bo_prng;
 
 	struct timespec repeat_start;
 

@@ -779,10 +786,12 @@ static int add_buffers(struct working_set *set, char *str)
 	 * 4m
 	 * 4g
 	 * 10n4k - 10 4k batches
+	 * 4096-16k - random size in range
 	 */
-	unsigned long *sizes, size;
+	struct work_buffer_size *sizes;
+	unsigned long min_sz, max_sz;
+	char *n, *max = NULL;
 	int add, i;
-	char *n;
 
 	n = index(str, 'n');
 	if (n) {

@@ -795,16 +804,34 @@ static int add_buffers(struct working_set *set, char *str)
 		add = 1;
 	}
 
-	size = parse_size(str);
-	if (!size)
+	n = index(str, '-');
+	if (n) {
+		*n = 0;
+		max = ++n;
+	}
+
+	min_sz = parse_size(str);
+	if (!min_sz)
 		return -1;
 
+	if (max) {
+		max_sz = parse_size(max);
+		if (!max_sz)
+			return -1;
+	} else {
+		max_sz = min_sz;
+	}
+
 	sizes = realloc(set->sizes, (set->nr + add) * sizeof(*sizes));
 	if (!sizes)
 		return -1;
 
-	for (i = 0; i < add; i++)
-		sizes[set->nr + i] = size;
+	for (i = 0; i < add; i++) {
+		struct work_buffer_size *sz = &sizes[set->nr + i];
+		sz->min = min_sz;
+		sz->max = max_sz;
+		sz->size = 0;
+	}
 
 	set->nr += add;
 	set->sizes = sizes;

@@ -846,7 +873,7 @@ static uint64_t engine_list_mask(const char *_str)
 	return mask;
 }
 
-static void allocate_working_set(struct working_set *set);
+static void allocate_working_set(struct workload *wrk, struct working_set *set);
 
 #define int_field(_STEP_, _FIELD_, _COND_, _ERR_) \
 	if ((field = strtok_r(fstart, ".", &fctx))) { \

@@ -1203,6 +1230,7 @@ add_step:
 	wrk->sseu = arg->sseu;
 	wrk->max_working_set_id = -1;
 	wrk->working_sets = NULL;
+	wrk->bo_prng = (flags & SYNCEDCLIENTS) ? master_prng : rand();
 
 	free(desc);
 

@@ -1256,7 +1284,7 @@ add_step:
 	 */
 	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
 		if (w->type == WORKINGSET && w->working_set.shared)
-			allocate_working_set(&w->working_set);
+			allocate_working_set(wrk, &w->working_set);
 	}
 
 	wrk->max_working_set_id = -1;

@@ -1533,7 +1561,7 @@ alloc_step_batch(struct workload *wrk, struct w_step *w)
 
 			igt_assert(set->nr);
 			igt_assert(entry->target < set->nr);
-			igt_assert(set->sizes[entry->target]);
+			igt_assert(set->sizes[entry->target].size);
 
 			dep_handle = set->handles[entry->target];
 		}

@@ -1701,15 +1729,27 @@ static size_t sizeof_engines_bond(int count)
 			engines[count]);
 }
 
-static void allocate_working_set(struct working_set *set)
+static unsigned long
+get_buffer_size(struct workload *wrk, const struct work_buffer_size *sz)
+{
+	if (sz->min == sz->max)
+		return sz->min;
+	else
+		return sz->min + hars_petruska_f54_1_random(&wrk->bo_prng) %
+		       (sz->max + 1 - sz->min);
+}
+
+static void allocate_working_set(struct workload *wrk, struct working_set *set)
 {
 	unsigned int i;
 
 	set->handles = calloc(set->nr, sizeof(*set->handles));
 	igt_assert(set->handles);
 
-	for (i = 0; i < set->nr; i++)
-		set->handles[i] = alloc_bo(fd, set->sizes[i]);
+	for (i = 0; i < set->nr; i++) {
+		set->sizes[i].size = get_buffer_size(wrk, &set->sizes[i]);
+		set->handles[i] = alloc_bo(fd, set->sizes[i].size);
+	}
 }
 
 #define alloca0(sz) ({ size_t sz__ = (sz); memset(alloca(sz__), 0, sz__); })

@@ -1724,6 +1764,7 @@ static int prepare_workload(unsigned int id, struct workload *wrk)
 
 	wrk->id = id;
 	wrk->bb_prng = (wrk->flags & SYNCEDCLIENTS) ? master_prng : rand();
+	wrk->bo_prng = (wrk->flags & SYNCEDCLIENTS) ? master_prng : rand();
 	wrk->run = true;
 
 	/*

@@ -1956,7 +1997,7 @@ static int prepare_workload(unsigned int id, struct workload *wrk)
 	 */
 	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
 		if (w->type == WORKINGSET && !w->working_set.shared)
-			allocate_working_set(&w->working_set);
+			allocate_working_set(wrk, &w->working_set);
 	}
 
 	/*

index a387e180c242..2d6d0a6a7b4b 100644
@@ -41,6 +41,7 @@
 #include <assert.h>
 #include <limits.h>
 #include <pthread.h>
+#include <math.h>
 
 #include "intel_chipset.h"
 #include "intel_reg.h"

@@ -875,6 +876,11 @@ static uint64_t engine_list_mask(const char *_str)
 
 static void allocate_working_set(struct workload *wrk, struct working_set *set);
 
+static long __duration(long dur, double scale)
+{
+	return round(scale * dur);
+}
+
 #define int_field(_STEP_, _FIELD_, _COND_, _ERR_) \
 	if ((field = strtok_r(fstart, ".", &fctx))) { \
 		tmp = atoi(field); \

@@ -885,7 +891,8 @@ static void allocate_working_set(struct workload *wrk, struct working_set *set);
 	} \
 
 static struct workload *
-parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
+parse_workload(struct w_arg *arg, unsigned int flags, double scale_dur,
+	       double scale_time, struct workload *app_w)
 {
 	struct workload *wrk;
 	unsigned int nr_steps = 0;

@@ -1151,7 +1158,7 @@ parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
 					  tmpl == LONG_MAX,
 					  "Invalid duration at step %u!\n",
 					  nr_steps);
-				step.duration.min = tmpl;
+				step.duration.min = __duration(tmpl, scale_dur);
 
 				if (sep && *sep == '-') {
 					tmpl = strtol(sep + 1, NULL, 10);

@@ -1161,7 +1168,8 @@ parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
 						tmpl == LONG_MAX,
 						"Invalid duration range at step %u!\n",
 						nr_steps);
-					step.duration.max = tmpl;
+					step.duration.max = __duration(tmpl,
+								       scale_dur);
 				} else {
 					step.duration.max = step.duration.min;
 				}

@@ -1197,6 +1205,9 @@ parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
 		step.type = BATCH;
 
 add_step:
+		if (step.type == DELAY)
+			step.delay = __duration(step.delay, scale_time);
+
 		step.idx = nr_steps++;
 		step.request = -1;
 		steps = realloc(steps, sizeof(step) * nr_steps);

@@ -2508,7 +2519,9 @@ static void print_help(void)
 "                    command line. Subsequent -s switches it off.\n"
 "  -S                Synchronize the sequence of random batch durations between\n"
 "                    clients.\n"
-"  -d                Sync between data dependencies in userspace."
+"  -d                Sync between data dependencies in userspace.\n"
+"  -f <scale>        Scale factor for batch durations.\n"
+"  -F <scale>        Scale factor for delays."
 	);
 }
 

@@ -2570,6 +2583,8 @@ int main(int argc, char **argv)
 	struct w_arg *w_args = NULL;
 	unsigned int tolerance_pct = 1;
 	int exitcode = EXIT_FAILURE;
+	double scale_time = 1.0f;
+	double scale_dur = 1.0f;
 	int prio = 0;
 	double t;
 	int i, c;

@@ -2590,7 +2605,7 @@ int main(int argc, char **argv)
 	master_prng = time(NULL);
 
 	while ((c = getopt(argc, argv,
-			   "ThqvsSdc:n:r:w:W:a:t:p:I:")) != -1) {
+			   "ThqvsSdc:n:r:w:W:a:t:p:I:f:F:")) != -1) {
 		switch (c) {
 		case 'W':
 			if (master_workload >= 0) {

@@ -2701,6 +2716,12 @@ int main(int argc, char **argv)
 		case 'I':
 			master_prng = strtol(optarg, NULL, 0);
 			break;
+		case 'f':
+			scale_dur = atof(optarg);
+			break;
+		case 'F':
+			scale_time = atof(optarg);
+			break;
 		case 'h':
 			print_help();
 			goto out;

@@ -2758,7 +2779,8 @@ int main(int argc, char **argv)
 
 	if (append_workload_arg) {
 		struct w_arg arg = { NULL, append_workload_arg, 0 };
-		app_w = parse_workload(&arg, flags, NULL);
+		app_w = parse_workload(&arg, flags, scale_dur, scale_time,
+				       NULL);
 		if (!app_w) {
 			wsim_err("Failed to parse append workload!\n");
 			goto err;

@@ -2776,7 +2798,8 @@ int main(int argc, char **argv)
 			goto err;
 		}
 
-		wrk[i] = parse_workload(&w_args[i], flags, app_w);
+		wrk[i] = parse_workload(&w_args[i], flags, scale_dur,
+					scale_time, app_w);
 		if (!wrk[i]) {
 			wsim_err("Failed to parse workload %u!\n", i);
 			goto err;

index 2d6d0a6a7b4b..8788f752121b 100644
@@ -874,7 +874,8 @@ static uint64_t engine_list_mask(const char *_str)
 	return mask;
 }
 
-static void allocate_working_set(struct workload *wrk, struct working_set *set);
+static unsigned long
+allocate_working_set(struct workload *wrk, struct working_set *set);
 
 static long __duration(long dur, double scale)
 {

@@ -1294,8 +1295,14 @@ add_step:
 	 * Allocate shared working sets.
 	 */
 	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
-		if (w->type == WORKINGSET && w->working_set.shared)
-			allocate_working_set(wrk, &w->working_set);
+		if (w->type == WORKINGSET && w->working_set.shared) {
+			unsigned long total =
+				allocate_working_set(wrk, &w->working_set);
+
+			if (verbose > 1)
+				printf("%u: %lu bytes in shared working set %u\n",
+				       wrk->id, total, w->working_set.id);
+		}
 	}
 
 	wrk->max_working_set_id = -1;

@@ -1750,8 +1757,10 @@ get_buffer_size(struct workload *wrk, const struct work_buffer_size *sz)
 		       (sz->max + 1 - sz->min);
 }
 
-static void allocate_working_set(struct workload *wrk, struct working_set *set)
+static unsigned long
+allocate_working_set(struct workload *wrk, struct working_set *set)
 {
+	unsigned long total = 0;
 	unsigned int i;
 
 	set->handles = calloc(set->nr, sizeof(*set->handles));

@@ -1760,7 +1769,82 @@ static void allocate_working_set(struct workload *wrk, struct working_set *set)
 	for (i = 0; i < set->nr; i++) {
 		set->sizes[i].size = get_buffer_size(wrk, &set->sizes[i]);
 		set->handles[i] = alloc_bo(fd, set->sizes[i].size);
+		total += set->sizes[i].size;
+	}
+
+	return total;
+}
+
+static bool
+find_dep(struct dep_entry *deps, unsigned int nr, struct dep_entry dep)
+{
+	unsigned int i;
+
+	for (i = 0; i < nr; i++) {
+		if (deps[i].working_set == dep.working_set &&
+		    deps[i].target == dep.target)
+			return true;
 	}
+
+	return false;
+}
+
+static void measure_active_set(struct workload *wrk)
+{
+	unsigned long total = 0, batch_sizes = 0;
+	struct dep_entry *deps = NULL;
+	unsigned int nr = 0, i, j;
+	struct w_step *w;
+
+	if (verbose < 3)
+		return;
+
+	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
+		if (w->type != BATCH)
+			continue;
+
+		batch_sizes += w->bb_sz;
+
+		for (j = 0; j < w->data_deps.nr; j++) {
+			struct dep_entry *dep = &w->data_deps.list[j];
+			struct dep_entry _dep = *dep;
+
+			if (dep->working_set == -1 && dep->target < 0) {
+				int idx = w->idx + dep->target;
+
+				igt_assert(idx >= 0 && idx < w->idx);
+				igt_assert(wrk->steps[idx].type == BATCH);
+
+				_dep.target = wrk->steps[idx].obj[0].handle;
+			}
+
+			if (!find_dep(deps, nr, _dep)) {
+				if (dep->working_set == -1) {
+					total += 4096;
+				} else {
+					struct working_set *set;
+
+					igt_assert(dep->working_set <=
+						   wrk->max_working_set_id);
+
+					set = wrk->working_sets[dep->working_set];
+					igt_assert(set->nr);
+					igt_assert(dep->target < set->nr);
+					igt_assert(set->sizes[dep->target].size);
+
+					total += set->sizes[dep->target].size;
+				}
+
+				deps = realloc(deps, (nr + 1) * sizeof(*deps));
+				deps[nr++] = *dep;
+			}
+		}
+	}
+
+	free(deps);
+
+	printf("%u: %lu bytes active working set in %u buffers. %lu in batch buffers.\n",
+	       wrk->id, total, nr, batch_sizes);
 }
 
 #define alloca0(sz) ({ size_t sz__ = (sz); memset(alloca(sz__), 0, sz__); })

@@ -1768,6 +1852,7 @@ static void allocate_working_set(struct workload *wrk, struct working_set *set)
 static int prepare_workload(unsigned int id, struct workload *wrk)
 {
 	struct working_set **sets;
+	unsigned long total = 0;
 	uint32_t share_vm = 0;
 	int max_ctx = -1;
 	struct w_step *w;

@@ -2008,9 +2093,12 @@ static int prepare_workload(unsigned int id, struct workload *wrk)
 	 */
 	for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) {
 		if (w->type == WORKINGSET && !w->working_set.shared)
-			allocate_working_set(wrk, &w->working_set);
+			total += allocate_working_set(wrk, &w->working_set);
 	}
 
+	if (verbose > 2)
+		printf("%u: %lu bytes in working sets.\n", wrk->id, total);
+
 	/*
 	 * Map of working set ids.
 	 */

@@ -2058,6 +2146,8 @@ static int prepare_workload(unsigned int id, struct workload *wrk)
 		alloc_step_batch(wrk, w);
 	}
 
+	measure_active_set(wrk);
+
 	return 0;
 }
 

index 8788f752121b..59ddc798a3ea 100644
@@ -43,6 +43,7 @@
 #include <pthread.h>
 #include <math.h>
 
+#include "igt_device_scan.h"
 #include "intel_chipset.h"
 #include "intel_reg.h"
 #include "drm.h"

@@ -2611,7 +2612,9 @@ static void print_help(void)
 "                    clients.\n"
 "  -d                Sync between data dependencies in userspace.\n"
 "  -f <scale>        Scale factor for batch durations.\n"
-"  -F <scale>        Scale factor for delays."
+"  -F <scale>        Scale factor for delays.\n"
+"  -L                List GPUs.\n"
+"  -D <gpu>          One of the GPUs from -L."
 	);
 }
 

@@ -2672,31 +2675,32 @@ int main(int argc, char **argv)
 	char *append_workload_arg = NULL;
 	struct w_arg *w_args = NULL;
 	unsigned int tolerance_pct = 1;
+	enum igt_devices_print_type printtype = IGT_PRINT_SIMPLE;
+	bool list_devices_arg = false;
 	int exitcode = EXIT_FAILURE;
+	struct igt_device_card card;
 	double scale_time = 1.0f;
 	double scale_dur = 1.0f;
+	char *device_arg = NULL;
 	int prio = 0;
 	double t;
-	int i, c;
+	int i, c, ret;
 	char *subopts, *value;
 	int raw_number = 0;
 	long calib_val;
 	int eng;
 
-	/*
-	 * Open the device via the low-level API so we can do the GPU quiesce
-	 * manually as close as possible in time to the start of the workload.
-	 * This minimizes the gap in engine utilization tracking when observed
-	 * via external tools like trace.pl.
-	 */
-	fd = __drm_open_driver_render(DRIVER_INTEL);
-	igt_require(fd);
-
 	master_prng = time(NULL);
 
 	while ((c = getopt(argc, argv,
-			   "ThqvsSdc:n:r:w:W:a:t:p:I:f:F:")) != -1) {
+			   "LThqvsSdc:n:r:w:W:a:t:p:I:f:F:D:")) != -1) {
 		switch (c) {
+		case 'L':
+			list_devices_arg = true;
+			break;
+		case 'D':
+			device_arg = strdup(optarg);
+			break;
 		case 'W':
 			if (master_workload >= 0) {
 				wsim_err("Only one master workload can be given!\n");

@@ -2820,6 +2824,33 @@ int main(int argc, char **argv)
 		}
 	}
 
+
+	igt_devices_scan(false);
+
+	if (list_devices_arg) {
+		igt_devices_print(printtype);
+		return EXIT_SUCCESS;
+	}
+
+	if (device_arg) {
+		ret = igt_device_card_match(device_arg, &card);
+		if (!ret) {
+			wsim_err("Requested device %s not found!\n", device_arg);
+			free(device_arg);
+
+			return EXIT_FAILURE;
+		}
+		free(device_arg);
+	} else {
+		igt_device_find_first_i915_discrete_card(&card);
+	}
+
+	if (card.render[0])
+		fd = igt_open_render(&card);
+	else
+		fd = __drm_open_driver_render(DRIVER_INTEL);
+	igt_require(fd);
+
 	if (!has_nop_calibration) {
 		if (verbose > 1) {
 			printf("Calibrating nop delays with %u%% tolerance...\n",

@@ -2932,15 +2963,12 @@ int main(int argc, char **argv)
 	clock_gettime(CLOCK_MONOTONIC, &t_start);
 
 	for (i = 0; i < clients; i++) {
-		int ret;
-
 		ret = pthread_create(&w[i]->thread, NULL, run_workload, w[i]);
 		igt_assert_eq(ret, 0);
 	}
 
 	if (master_workload >= 0) {
-		int ret = pthread_join(w[master_workload]->thread, NULL);
-
+		ret = pthread_join(w[master_workload]->thread, NULL);
 		igt_assert(ret == 0);
 
 		for (i = 0; i < clients; i++)

@@ -2949,7 +2977,7 @@ int main(int argc, char **argv)
 
 	for (i = 0; i < clients; i++) {
 		if (master_workload != i) {
-			int ret = pthread_join(w[i]->thread, NULL);
+			ret = pthread_join(w[i]->thread, NULL);
 			igt_assert(ret == 0);
 		}
 	}

index 59ddc798a3ea..811a4b1b7161 100644
@@ -296,8 +296,8 @@ print_engine_calibrations(void)
 
 	printf("Nop calibration for %uus delay is: ", nop_calibration_us);
 	for (int i = 0; i < NUM_ENGINES; i++) {
-		/* skip DEFAULT and VCS engines */
-		if (i != DEFAULT && i != VCS) {
+		/* skip engines not present and DEFAULT and VCS */
+		if (i != DEFAULT && i != VCS && engine_calib_map[i]) {
 			if (first_entry) {
 				printf("%s=%lu", ring_str_map[i], engine_calib_map[i]);
 				first_entry = false;

@@ -2862,22 +2862,6 @@ int main(int argc, char **argv)
 		if (verbose)
 			print_engine_calibrations();
 		goto out;
-	} else {
-		bool missing = false;
-
-		for (i = 0; i < NUM_ENGINES; i++) {
-			if (i == VCS)
-				continue;
-
-			if (!engine_calib_map[i]) {
-				wsim_err("Missing calibration for '%s'!\n",
-					 ring_str_map[i]);
-				missing = true;
-			}
-		}
-
-		if (missing)
-			goto err;
 	}
 
 	if (!nr_w_args) {

index 811a4b1b7161..496e5042d89c 100644
@@ -1449,6 +1449,7 @@ static unsigned int terminate_bb(struct w_step *w)
 	const uint32_t bbe = 0xa << 23;
 	unsigned long mmap_start, mmap_len;
 	unsigned long batch_start = w->bb_sz;
+	bool keep_mmap = false;
 	unsigned int r = 0;
 	uint32_t *ptr, *cs;
 

@@ -1472,6 +1473,7 @@ static unsigned int terminate_bb(struct w_step *w)
 
 		*cs++ = w->preempt_us ? 0x5 << 23 /* MI_ARB_CHK; */ : MI_NOOP;
 		w->recursive_bb_start = cs;
+		keep_mmap = true;
 		*cs++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
 		*cs++ = 0;
 		*cs++ = 0;

@@ -1479,6 +1481,9 @@ static unsigned int terminate_bb(struct w_step *w)
 
 	*cs = bbe;
 
+	if (!keep_mmap)
+		munmap(ptr, mmap_len);
+
 	return r;
 }