Skip to contentSkip to conversation

s390x: css: pv: css test adaptation for PV

PATCH Superseded Pierre Morel 2 participants 8 messages Sep 25, 2020 – Sep 28, 2020

Subsystems (3)

v1 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

cover s390x: css: pv: css test adaptation for PV

Hi all,

To adapt the CSS I/O tests to protected virtualisation we need utilities to:

1- allocate the I/O buffers in a private page (patch 3)
It must be in a dedicated page to avoid exporting code or guest private data to the host.

2- share the I/O buffer with the host (patch 2)

3- be sure to allocate memory under 2Gb (patch 1)

The 3 first patches are the implementation of the tools, patch 4 is the modification of the css.c test for PV.

regards,
Pierre

Pierre Morel (4):
memory: allocation in low memory
s390x: pv: implement routine to share/unshare memory
s390: define UV compatible I/O allocation
s390x: css: pv: css test adaptation for PV

 lib/alloc_dma_page.c  | 57 +++++++++++++++++++++++++++++++++++++++++++
 lib/alloc_dma_page.h  | 24 ++++++++++++++++++
 lib/s390x/asm/uv.h    | 33 +++++++++++++++++++++++++
 lib/s390x/css.h       |  3 +--
 lib/s390x/css_lib.c   | 28 ++++++---------------
 lib/s390x/malloc_io.c | 50 +++++++++++++++++++++++++++++++++++++
 lib/s390x/malloc_io.h | 18 ++++++++++++++
 lib/s390x/sclp.c      |  2 ++
 s390x/Makefile        |  2 ++
 s390x/css.c           | 35 +++++++++++++++++---------
 10 files changed, 219 insertions(+), 33 deletions(-)
 create mode 100644 lib/alloc_dma_page.c
 create mode 100644 lib/alloc_dma_page.h
 create mode 100644 lib/s390x/malloc_io.c
 create mode 100644 lib/s390x/malloc_io.h
Signature · 1 line
2.25.1
#1 lore raw

PATCH 1/4 memory: allocation in low memory

Some architectures need allocations to be done under a specific address limit to allow DMA from I/O.

We propose here a very simple page allocator to get pages allocated under this specific limit.

The DMA page allocator will only use part of the available memory under the DMA address limit to let room for the standard allocator.

---
 lib/alloc_dma_page.c | 57 ++++++++++++++++++++++++++++++++++++++++++++
 lib/alloc_dma_page.h | 24 +++++++++++++++++++
 lib/s390x/sclp.c     |  2 ++
 s390x/Makefile       |  1 +
 4 files changed, 84 insertions(+)
 create mode 100644 lib/alloc_dma_page.c
 create mode 100644 lib/alloc_dma_page.h
lib/alloc_dma_page.c new file +57 -0
new file mode 100644
index 0000000..6a16e38
@@ -0,0 +1,57 @@
+/*
+ * Page allocator for DMA
+ *
+ * Copyright (c) IBM, Corp. 2020
+ *
+ * Authors:
+ *  Pierre Morel <[email protected]>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Library General Public License version 2.
+ */
+#include <libcflat.h>
+#include <asm/spinlock.h>
+#include <alloc_dma_page.h>
+
+static struct spinlock lock;
+static void *dma_freelist = 0;
+
+void put_dma_page(void *dma_page)
+{
+	spin_lock(&lock);
+	*(void **)dma_page = dma_freelist;
+	dma_freelist = dma_page;
+	spin_unlock(&lock);
+}
+
+void *get_dma_page(void)
+{
+	void *p = NULL;
+
+	spin_lock(&lock);
+	if (!dma_freelist)
+		goto end_unlock;
+
+	p = dma_freelist;
+	dma_freelist = *(void **)dma_freelist;
+
+end_unlock:
+	spin_unlock(&lock);
+	return p;
+}
+
+phys_addr_t dma_page_alloc_init(phys_addr_t start, phys_addr_t end)
+{
+	int start_pfn = start >> PAGE_SHIFT;
+	int nb_pfn = ((end - start) >> PAGE_SHIFT) - 1;
+	int max, pfn;
+
+	max = start_pfn + nb_pfn / DMA_ALLOC_RATIO;
+	if (max > DMA_MAX_PFN)
+		max = DMA_MAX_PFN;
+
+	for (pfn = start_pfn; pfn < max; pfn++)
+		put_dma_page((void *)((unsigned long) pfn << PAGE_SHIFT));
+
+	return (phys_addr_t)pfn << PAGE_SHIFT;
+}
lib/alloc_dma_page.h new file +24 -0
new file mode 100644
index 0000000..85e1d2f
@@ -0,0 +1,24 @@
+/*
+ * Page allocator for DMA definitions
+ *
+ * Copyright (c) IBM, Corp. 2020
+ *
+ * Authors:
+ *  Pierre Morel <[email protected]>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Library General Public License version 2.
+ */
+#ifndef _ALLOC_DMA_PAGE_H_
+#define _ALLOC_DMA_PAGE_H_
+
+#include <asm/page.h>
+
+void put_dma_page(void *dma_page);
+void *get_dma_page(void);
+phys_addr_t dma_page_alloc_init(phys_addr_t start_pfn, phys_addr_t nb_pages);
+
+#define DMA_MAX_PFN	(0x80000000 >> PAGE_SHIFT)
+#define DMA_ALLOC_RATIO	8
+
+#endif /* _ALLOC_DMA_PAGE_H_ */
lib/s390x/sclp.c +2 -0
index 4054d0e..9c95ca5 100644
@@ -19,6 +19,7 @@
 #include "sclp.h"
 #include <alloc_phys.h>
 #include <alloc_page.h>
+#include <alloc_dma_page.h>
 
 extern unsigned long stacktop;
 
@@ -35,6 +36,7 @@ static void mem_init(phys_addr_t mem_end)
 	phys_addr_t freemem_start = (phys_addr_t)&stacktop;
 	phys_addr_t base, top;
 
+	freemem_start = dma_page_alloc_init(freemem_start, mem_end);
 	phys_alloc_init(freemem_start, mem_end - freemem_start);
 	phys_alloc_get_unused(&base, &top);
 	base = (base + PAGE_SIZE - 1) & -PAGE_SIZE;
s390x/Makefile +1 -0
index 9144d57..109ef9f 100644
@@ -52,6 +52,7 @@ cflatobjs += lib/alloc_phys.o
 cflatobjs += lib/alloc_page.o
 cflatobjs += lib/vmalloc.o
 cflatobjs += lib/alloc_phys.o
+cflatobjs += lib/alloc_dma_page.o
 cflatobjs += lib/s390x/io.o
 cflatobjs += lib/s390x/stack.o
 cflatobjs += lib/s390x/sclp.o
Signature · 1 line
2.25.1
#2 lore raw

Re: [PATCH v1 1/4] memory: allocation in low memory

Arrrghh sorry, I forgot the kvm-unit-tests TAG

Quoted 9 lines · Pierre Morel

Some architectures need allocations to be done under a specific address limit to allow DMA from I/O.

We propose here a very simple page allocator to get pages allocated under this specific limit.

The DMA page allocator will only use part of the available memory under the DMA address limit to let room for the standard allocator.

Signature · 2 lines
Pierre Morel
IBM Lab Boeblingen
#3 lore raw

PATCH 2/4 s390x: pv: implement routine to share/unshare memory

In reply to Pierre Morel

When communicating with the host we need to share part of the memory.

Let's implement the ultravisor calls for this.

---
 lib/s390x/asm/uv.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
lib/s390x/asm/uv.h +33 -0
index 4c2fc48..19019e4 100644
@@ -71,4 +71,37 @@ static inline int uv_call(unsigned long r1, unsigned long r2)
 	return cc;
 }
 
+static inline int share(unsigned long addr, u16 cmd)
+{
+	struct uv_cb_share uvcb = {
+		.header.cmd = cmd,
+		.header.len = sizeof(uvcb),
+		.paddr = addr
+	};
+
+	uv_call(0, (u64)&uvcb);
+	return uvcb.header.rc;
+}
+
+/*
+ * Guest 2 request to the Ultravisor to make a page shared with the
+ * hypervisor for IO.
+ *
+ * @addr: Real or absolute address of the page to be shared
+ */
+static inline int uv_set_shared(unsigned long addr)
+{
+	return share(addr, UVC_CMD_SET_SHARED_ACCESS);
+}
+
+/*
+ * Guest 2 request to the Ultravisor to make a page unshared.
+ *
+ * @addr: Real or absolute address of the page to be unshared
+ */
+static inline int uv_remove_shared(unsigned long addr)
+{
+	return share(addr, UVC_CMD_REMOVE_SHARED_ACCESS);
+}
+
 #endif
Signature · 1 line
2.25.1
#4 lore raw

Re: [PATCH v1 2/4] s390x: pv: implement routine to share/unshare memory

Quoted 19 lines · Pierre Morel

When communicating with the host we need to share part of the memory.

Let's implement the ultravisor calls for this.

---
 lib/s390x/asm/uv.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
lib/s390x/asm/uv.h +0 -0
index 4c2fc48..19019e4 100644
@@ -71,4 +71,37 @@ static inline int uv_call(unsigned long r1, unsigned long r2)
 	return cc;
 }
 

lib/s390x/asm/uv.h static inline int uv_call(unsigned long r1, unsigned long r2) · line 83

Quoted 10 lines
lib/s390x/asm/uv.h +10 -0
+static inline int share(unsigned long addr, u16 cmd)
+{
+	struct uv_cb_share uvcb = {
+		.header.cmd = cmd,
+		.header.len = sizeof(uvcb),
+		.paddr = addr
+	};
+
+	uv_call(0, (u64)&uvcb);
+	return uvcb.header.rc;

That's not a great idea, rc is > 1 for error codes...
In the kernel we check for the cc instead since uv_call() has only 0/1 as possible cc return values.

Quoted 25 lines
lib/s390x/asm/uv.h +23 -0
+}
+
+/*
+ * Guest 2 request to the Ultravisor to make a page shared with the
+ * hypervisor for IO.
+ *
+ * @addr: Real or absolute address of the page to be shared
+ */
+static inline int uv_set_shared(unsigned long addr)
+{
+	return share(addr, UVC_CMD_SET_SHARED_ACCESS);
+}
+
+/*
+ * Guest 2 request to the Ultravisor to make a page unshared.
+ *
+ * @addr: Real or absolute address of the page to be unshared
+ */
+static inline int uv_remove_shared(unsigned long addr)
+{
+	return share(addr, UVC_CMD_REMOVE_SHARED_ACCESS);
+}
+
 #endif
#5 lore raw

Re: [PATCH v1 2/4] s390x: pv: implement routine to share/unshare memory

Quoted 20 lines · Janosch Frank
Quoted 19 lines · Pierre Morel

When communicating with the host we need to share part of the memory.

Let's implement the ultravisor calls for this.

---
  lib/s390x/asm/uv.h | 33 +++++++++++++++++++++++++++++++++
  1 file changed, 33 insertions(+)
lib/s390x/asm/uv.h +0 -0
index 4c2fc48..19019e4 100644
@@ -71,4 +71,37 @@ static inline int uv_call(unsigned long r1, unsigned long r2)
  	return cc;
  }
 
Quoted 14 lines

lib/s390x/asm/uv.h static inline int uv_call(unsigned long r1, unsigned long r2) · line 83

Quoted 10 lines
lib/s390x/asm/uv.h +10 -0
+static inline int share(unsigned long addr, u16 cmd)
+{
+	struct uv_cb_share uvcb = {
+		.header.cmd = cmd,
+		.header.len = sizeof(uvcb),
+		.paddr = addr
+	};
+
+	uv_call(0, (u64)&uvcb);
+	return uvcb.header.rc;

That's not a great idea, rc is > 1 for error codes...
In the kernel we check for the cc instead since uv_call() has only 0/1 as possible cc return values.

hum, Yes.
I will add some report too here in case of error.

Signature · 2 lines
Pierre Morel
IBM Lab Boeblingen
#6 lore raw

PATCH 3/4 s390: define UV compatible I/O allocation

In reply to Pierre Morel

To centralize the memory allocation for I/O we define the alloc/free_io_page() functions which share the I/O memory with the host in case the guest runs with protected virtualization.

---
 lib/s390x/malloc_io.c | 50 +++++++++++++++++++++++++++++++++++++++++++
 lib/s390x/malloc_io.h | 18 ++++++++++++++++
 s390x/Makefile        |  1 +
 3 files changed, 69 insertions(+)
 create mode 100644 lib/s390x/malloc_io.c
 create mode 100644 lib/s390x/malloc_io.h
lib/s390x/malloc_io.c new file +50 -0
new file mode 100644
index 0000000..b84a06a
@@ -0,0 +1,50 @@
+/*
+ * I/O page allocation
+ *
+ * Copyright (c) 2020 IBM Corp
+ *
+ * Authors:
+ *  Pierre Morel <[email protected]>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2.
+ *
+ * Using this interface provide host access to the allocated pages in
+ * case the guest is a secure guest.
+ * This is needed for I/O buffers.
+ *
+ */
+#include <libcflat.h>
+#include <alloc_dma_page.h>
+#include <asm/uv.h>
+#include <malloc_io.h>
+#include <asm/facility.h>
+
+void *alloc_io_page(int size)
+{
+	void *p;
+
+	assert(size <= PAGE_SIZE);
+	p = get_dma_page();
+	if (!p) {
+		report(0, "Memory allocation");
+		return NULL;
+	}
+
+	if (!test_facility(158))
+		return p;
+
+	if (!uv_set_shared((unsigned long)p)) {
+		report(0, "Sharing memory");
+		return NULL;
+	}
+
+	return p;
+}
+
+void free_io_page(void *p)
+{
+	if (test_facility(158) && !uv_remove_shared((unsigned long)p))
+		report(0, "Unsharing memory");
+	put_dma_page(p);
+}
lib/s390x/malloc_io.h new file +18 -0
new file mode 100644
index 0000000..c6ed481
@@ -0,0 +1,18 @@
+/*
+ * I/O allocations
+ *
+ * Copyright (c) 2020 IBM Corp
+ *
+ * Authors:
+ *  Pierre Morel <[email protected]>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2.
+ */
+#ifndef _S390X_MALLOC_IO_H_
+#define _S390X_MALLOC_IO_H_
+
+void *alloc_io_page(int size);
+void free_io_page(void *p);
+
+#endif /* _S390X_MALLOC_IO_H_ */
s390x/Makefile +1 -0
index 109ef9f..1b1dc9f 100644
@@ -63,6 +63,7 @@ cflatobjs += lib/s390x/smp.o
 cflatobjs += lib/s390x/vm.o
 cflatobjs += lib/s390x/css_dump.o
 cflatobjs += lib/s390x/css_lib.o
+cflatobjs += lib/s390x/malloc_io.o
 
 OBJDIRS += lib/s390x
 
Signature · 1 line
2.25.1
#7 lore raw

PATCH 4/4 s390x: css: pv: css test adaptation for PV

In reply to Pierre Morel

We want the tests to automatically work with or without protected virtualisation.
To do this we need to share the I/O memory with the host.

Let's replace all static allocations with dynamic allocations to clearly separate shared and private memory.

---
 lib/s390x/css.h     |  3 +--
 lib/s390x/css_lib.c | 28 ++++++++--------------------
 s390x/css.c         | 35 ++++++++++++++++++++++++-----------
 3 files changed, 33 insertions(+), 33 deletions(-)
lib/s390x/css.h +1 -2
index 221b67c..e3dee9f 100644
@@ -283,8 +283,7 @@ int css_enable(int schid, int isc);
 
 /* Library functions */
 int start_ccw1_chain(unsigned int sid, struct ccw1 *ccw);
-int start_single_ccw(unsigned int sid, int code, void *data, int count,
-		     unsigned char flags);
+struct ccw1 *ccw_alloc(int code, void *data, int count, unsigned char flags);
 void css_irq_io(void);
 int css_residual_count(unsigned int schid);
 
lib/s390x/css_lib.c +8 -20
index 8e02371..6a0a0ec 100644
@@ -18,6 +18,7 @@
 #include <asm/time.h>
 #include <asm/arch_def.h>
 
+#include <malloc_io.h>
 #include <css.h>
 
 static struct schib schib;
@@ -202,33 +203,20 @@ int start_ccw1_chain(unsigned int sid, struct ccw1 *ccw)
 	return ssch(sid, &orb);
 }
 
-/*
- * In the future, we want to implement support for CCW chains;
- * for that, we will need to work with ccw1 pointers.
- */
-static struct ccw1 unique_ccw;
-
-int start_single_ccw(unsigned int sid, int code, void *data, int count,
-		     unsigned char flags)
+struct ccw1 *ccw_alloc(int code, void *data, int count, unsigned char flags)
 {
-	int cc;
-	struct ccw1 *ccw = &unique_ccw;
+	struct ccw1 *ccw;
+
+	ccw = alloc_io_page(sizeof(*ccw));
+	if (!ccw)
+		return NULL;
 
-	report_prefix_push("start_subchannel");
-	/* Build the CCW chain with a single CCW */
 	ccw->code = code;
 	ccw->flags = flags;
 	ccw->count = count;
 	ccw->data_address = (int)(unsigned long)data;
 
-	cc = start_ccw1_chain(sid, ccw);
-	if (cc) {
-		report(0, "cc = %d", cc);
-		report_prefix_pop();
-		return cc;
-	}
-	report_prefix_pop();
-	return 0;
+	return ccw;
 }
 
 /* wait_and_check_io_completion:
s390x/css.c +24 -11
index ee3bc83..4b0b6b1 100644
@@ -17,13 +17,15 @@
 #include <interrupt.h>
 #include <asm/arch_def.h>
 
+#include <malloc_io.h>
 #include <css.h>
+#include <asm/barrier.h>
 
 #define DEFAULT_CU_TYPE		0x3832 /* virtio-ccw */
 static unsigned long cu_type = DEFAULT_CU_TYPE;
 
 static int test_device_sid;
-static struct senseid senseid;
+static struct senseid *senseid;
 
 static void test_enumerate(void)
 {
@@ -57,6 +59,7 @@ static void test_enable(void)
  */
 static void test_sense(void)
 {
+	struct ccw1 *ccw;
 	int ret;
 	int len;
 
@@ -80,9 +83,15 @@ static void test_sense(void)
 
 	lowcore_ptr->io_int_param = 0;
 
-	memset(&senseid, 0, sizeof(senseid));
-	ret = start_single_ccw(test_device_sid, CCW_CMD_SENSE_ID,
-			       &senseid, sizeof(senseid), CCW_F_SLI);
+	senseid = alloc_io_page(sizeof(*senseid));
+	if (!senseid)
+		goto error_senseid;
+
+	ccw = ccw_alloc(CCW_CMD_SENSE_ID, senseid, sizeof(*senseid), CCW_F_SLI);
+	if (!ccw)
+		goto error_ccw;
+
+	ret = start_ccw1_chain(test_device_sid, ccw);
 	if (ret)
 		goto error;
 
@@ -97,7 +106,7 @@ static void test_sense(void)
 	if (ret < 0) {
 		report_info("no valid residual count");
 	} else if (ret != 0) {
-		len = sizeof(senseid) - ret;
+		len = sizeof(*senseid) - ret;
 		if (ret && len < CSS_SENSEID_COMMON_LEN) {
 			report(0, "transferred a too short length: %d", ret);
 			goto error;
@@ -105,21 +114,25 @@ static void test_sense(void)
 			report_info("transferred a shorter length: %d", len);
 	}
 
-	if (senseid.reserved != 0xff) {
-		report(0, "transferred garbage: 0x%02x", senseid.reserved);
+	if (senseid->reserved != 0xff) {
+		report(0, "transferred garbage: 0x%02x", senseid->reserved);
 		goto error;
 	}
 
 	report_prefix_pop();
 
 	report_info("reserved 0x%02x cu_type 0x%04x cu_model 0x%02x dev_type 0x%04x dev_model 0x%02x",
-		    senseid.reserved, senseid.cu_type, senseid.cu_model,
-		    senseid.dev_type, senseid.dev_model);
+		    senseid->reserved, senseid->cu_type, senseid->cu_model,
+		    senseid->dev_type, senseid->dev_model);
 
-	report(senseid.cu_type == cu_type, "cu_type expected 0x%04x got 0x%04x",
-	       (uint16_t) cu_type, senseid.cu_type);
+	report(senseid->cu_type == cu_type, "cu_type expected 0x%04x got 0x%04x",
+	       (uint16_t) cu_type, senseid->cu_type);
 
 error:
+	free_io_page(ccw);
+error_ccw:
+	free_io_page(senseid);
+error_senseid:
 	unregister_io_int_func(css_irq_io);
 }
 
Signature · 1 line
2.25.1