Skip to contentSkip to conversation

windows/netuio: add Windows NetUIO kernel driver

PATCH Superseded Narcisa Ana Maria Vasile 4 participants 16 messages Aug 20, 2020 – Oct 14, 2020

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 windows/netuio: add Windows NetUIO kernel driver

From: Narcisa Vasile <[email protected]>

The Windows NetUIO kernel driver allows the DPDK userspace application to directly access the hardware.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>

---

v2:
Fix license message in each file
Minor fixes in README
Remove RC file as it is autogenerated
Remove HW Ids from Inf
Resubmit as single patch

 .gitattributes                                |   4 +
 .gitignore                                    |   2 +
 windows/netuio/kernel/README_NetUIO.rst       |  64 +++
 .../netuio/kernel/windows/netuio/netuio.inf   |  78 ++++
 .../netuio/kernel/windows/netuio/netuio_dev.c | 388 +++++++++++++++++
 .../netuio/kernel/windows/netuio/netuio_dev.h |  61 +++
 .../netuio/kernel/windows/netuio/netuio_drv.c | 146 +++++++
 .../netuio/kernel/windows/netuio/netuio_drv.h |  32 ++
 .../kernel/windows/netuio/netuio_interface.h  |  73 ++++
 .../kernel/windows/netuio/netuio_queue.c      | 397 ++++++++++++++++++
 .../kernel/windows/netuio/netuio_queue.h      |  31 ++
 .../netuio/kernel/windows/netuio/resource.h   |  14 +
 .../mk/exec-env/windows/netuio/netuio.sln     |  24 ++
 .../mk/exec-env/windows/netuio/netuio.vcxproj | 113 +++++
 .../windows/netuio/netuio.vcxproj.filters     |  54 +++
 .../windows/netuio/netuio.vcxproj.user        |  11 +
 16 files changed, 1492 insertions(+)
 create mode 100644 .gitattributes
 create mode 100644 .gitignore
 create mode 100644 windows/netuio/kernel/README_NetUIO.rst
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio.inf
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio_dev.c
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio_dev.h
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio_drv.c
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio_drv.h
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio_interface.h
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio_queue.c
 create mode 100644 windows/netuio/kernel/windows/netuio/netuio_queue.h
 create mode 100644 windows/netuio/kernel/windows/netuio/resource.h
 create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.sln
 create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj
 create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj.filters
 create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj.user
.gitattributes new file +4 -0
new file mode 100644
index 000000000..13482db3d
@@ -0,0 +1,4 @@
+* text=auto
+
+*.sln text eol=crlf
+*.vcxproj text eol=crlf
.gitignore new file +2 -0
new file mode 100644
index 000000000..543281e8f
@@ -0,0 +1,2 @@
+x64/
+.vs/
windows/netuio/kernel/README_NetUIO.rst new file +64 -0

64 lines not on this page · raw message

windows/netuio/kernel/windows/netuio/netuio.inf new file +78 -0

78 lines not on this page · raw message

windows/netuio/kernel/windows/netuio/netuio_dev.c new file +388 -0

388 lines not on this page · raw message

windows/netuio/kernel/windows/netuio/netuio_dev.h new file +61 -0

61 lines not on this page · raw message

windows/netuio/kernel/windows/netuio/netuio_drv.c new file +146 -0

146 lines not on this page · raw message

windows/netuio/kernel/windows/netuio/netuio_drv.h new file +32 -0
new file mode 100644
index 000000000..07743bf86
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) Microsoft Corporation. All rights reserved
+ */
+
+
+#ifndef NETUIO_DRV_H
+#define NETUIO_DRV_H
+
+#define INITGUID
+
+#include <ntddk.h>
+#include <wdf.h>
+
+#include "netuio_dev.h"
+#include "netuio_queue.h"
+
+EXTERN_C_START
+
+// Print output constants
+#define DPFLTR_NETUIO_INFO_LEVEL   35
+
+// WDFDRIVER Events
+DRIVER_INITIALIZE DriverEntry;
+EVT_WDF_DRIVER_DEVICE_ADD       netuio_evt_device_add;
+EVT_WDF_OBJECT_CONTEXT_CLEANUP  netuio_evt_driver_context_cleanup;
+EVT_WDF_DEVICE_PREPARE_HARDWARE netuio_evt_prepare_hw;
+EVT_WDF_DEVICE_RELEASE_HARDWARE netuio_evt_release_hw;
+EVT_WDF_FILE_CLOSE              netuio_evt_file_cleanup;
+
+EXTERN_C_END
+
+#endif // NETUIO_DRV_H
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +73 -0

73 lines not on this page · raw message

windows/netuio/kernel/windows/netuio/netuio_queue.c new file +397 -0

397 lines not on this page · raw message

windows/netuio/kernel/windows/netuio/netuio_queue.h new file +31 -0
new file mode 100644
index 000000000..9173a77b9
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) Microsoft Corporation. All rights reserved
+ */
+
+#ifndef NETUIO_QUEUE_H
+#define NETUIO_QUEUE_H
+
+EXTERN_C_START
+
+// This is the context that can be placed per queue and would contain per queue information.
+typedef struct _QUEUE_CONTEXT {
+    ULONG PrivateDeviceData;  // just a placeholder
+} QUEUE_CONTEXT, *PQUEUE_CONTEXT;
+
+WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(QUEUE_CONTEXT, QueueGetContext)
+
+VOID
+netuio_unmap_address_from_user_process(_In_ PNETUIO_CONTEXT_DATA netuio_contextdata);
+
+NTSTATUS
+netuio_queue_initialize(_In_ WDFDEVICE hDevice);
+
+// Events from the IoQueue object
+EVT_WDF_IO_QUEUE_IO_DEVICE_CONTROL netuio_evt_IO_device_control;
+EVT_WDF_IO_QUEUE_IO_STOP netuio_evt_IO_stop;
+
+EVT_WDF_IO_IN_CALLER_CONTEXT netuio_evt_IO_in_caller_context;
+
+EXTERN_C_END
+
+#endif // NETUIO_QUEUE_H
windows/netuio/kernel/windows/netuio/resource.h new file +14 -0
new file mode 100644
index 000000000..9789ffdf3
@@ -0,0 +1,14 @@
+//{{NO_DEPENDENCIES}}
+// Microsoft Visual C++ generated include file.
+// Used by netuio.rc
+
+// Next default values for new objects
+// 
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_NEXT_RESOURCE_VALUE        101
+#define _APS_NEXT_COMMAND_VALUE         40001
+#define _APS_NEXT_CONTROL_VALUE         1001
+#define _APS_NEXT_SYMED_VALUE           101
+#endif
+#endif
windows/netuio/mk/exec-env/windows/netuio/netuio.sln new file +24 -0
new file mode 100644
index 000000000..15c26e6b9
@@ -0,0 +1,24 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25420.1
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "netuio", "netuio.vcxproj", "{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.ActiveCfg = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Build.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Deploy.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.ActiveCfg = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Build.0 = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Deploy.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj new file +113 -0

114 lines not on this page · raw message

windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj.filters new file +54 -0

55 lines not on this page · raw message

windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj.user new file +11 -0
new file mode 100644
index 000000000..771ce8681
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <TestCertificate>
+    </TestCertificate>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <TestCertificate>
+    </TestCertificate>
+  </PropertyGroup>
+</Project>
\ No newline at end of file
Signature · 1 line
2.23.0.vfs.1.1.63.g5a5ad7f
#1 lore raw

Hi, Naty

Quoted 34 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows NetUIO kernel driver allows the DPDK userspace application to directly access the hardware.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>

---

v2:
Fix license message in each file
Minor fixes in README
Remove RC file as it is autogenerated
Remove HW Ids from Inf
Resubmit as single patch

  .gitattributes                                |   4 +
  .gitignore                                    |   2 +
  windows/netuio/kernel/README_NetUIO.rst       |  64 +++
  .../netuio/kernel/windows/netuio/netuio.inf   |  78 ++++
  .../netuio/kernel/windows/netuio/netuio_dev.c | 388 +++++++++++++++++
  .../netuio/kernel/windows/netuio/netuio_dev.h |  61 +++
  .../netuio/kernel/windows/netuio/netuio_drv.c | 146 +++++++
  .../netuio/kernel/windows/netuio/netuio_drv.h |  32 ++
  .../kernel/windows/netuio/netuio_interface.h  |  73 ++++
  .../kernel/windows/netuio/netuio_queue.c      | 397 ++++++++++++++++++
  .../kernel/windows/netuio/netuio_queue.h      |  31 ++
  .../netuio/kernel/windows/netuio/resource.h   |  14 +
  .../mk/exec-env/windows/netuio/netuio.sln     |  24 ++
  .../mk/exec-env/windows/netuio/netuio.vcxproj | 113 +++++
  .../windows/netuio/netuio.vcxproj.filters     |  54 +++
  .../windows/netuio/netuio.vcxproj.user        |  11 +
  16 files changed, 1492 insertions(+)
Quoted 16 lines

create mode 100644 .gitattributes
create mode 100644 .gitignore
create mode 100644 windows/netuio/kernel/README_NetUIO.rst
create mode 100644 windows/netuio/kernel/windows/netuio/netuio.inf create mode 100644 windows/netuio/kernel/windows/netuio/netuio_dev.c create mode 100644 windows/netuio/kernel/windows/netuio/netuio_dev.h create mode 100644 windows/netuio/kernel/windows/netuio/netuio_drv.c create mode 100644 windows/netuio/kernel/windows/netuio/netuio_drv.h create mode 100644 windows/netuio/kernel/windows/netuio/netuio_interface.h create mode 100644 windows/netuio/kernel/windows/netuio/netuio_queue.c create mode 100644 windows/netuio/kernel/windows/netuio/netuio_queue.h create mode 100644 windows/netuio/kernel/windows/netuio/resource.h create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.sln create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj.filters create mode 100644 windows/netuio/mk/exec-env/windows/netuio/netuio.vcxproj.user

I haven't reviewed the code yet, but a few general suggestions and observations:

1. This code is going into the dpdk-kmods repo, so we don't need the four-level deep directory path for the source.

2. The code can simply be in the windows/netuio directory (don't need 'kernel')

3. The 'virt2phys' module that is already in the repo puts the build-related files (project files, solution files etc) in the same folder as the source files. We should do the same here. We don't need the mk/exec-env subfolder for these files. They can be in the same directory as the source.

4. Remove the .vcxproj.user file from the patch. It is not required.

5. The README should be renamed to simply README.rst and it can also be in this windows/netuio folder

6. The existing README.rst in the windows folder contains information on how to build and load kernel modules in Windows. This new README can reference the root-level README, if required. (Also some of the information in this new README is out-of-date. I'll review that separately)

ranjit m.

#3 lore raw
In reply to Narcisa Ana Maria Vasile
Quoted 9 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows NetUIO kernel driver allows the DPDK userspace application to directly access the hardware.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>

---

Major questions:

1. Does NetUIO still need to allocate and map a contiguous memory segment now, when DPDK has user-mode memory management?

2. IOCTLs require to specify PCI address on each call. This is very inconvenient for DPDK consumers and also seems to serve no purpose.

3. There is a need to document driver's design, preferably in commit message, specifically:

3.1) DMA remapping capability in INF (AFAIK, vendors are notified); 3.2) manual BAR probing instead of using resource lists; 3.3) reason to use EvtIoInCallerContext and IO queues; 3.4) IOCTL format.

Also, I agree with everything Ranjit has noted already.

General suggestions to cleanup the code a bit. We can do it later if you wish. This also brings up the question, which code style should Windows kernel code for DPDK follow (off-topic for now).

* Remove boilerplate code and comments generated by VS wizard.
* Place _Use_decl_annotations on definitions to make them simpler.
* Limit line length, try using shorted variable names (e.g. "netuio_contextdata" may be "context" or "ctx" with no loss).

More specific comments inline.

Quoted 2 lines
 create mode 100644 .gitattributes
 create mode 100644 .gitignore

Both of these files should be in windows/ directory.

Quoted 1 line
 create mode 100644 windows/netuio/kernel/windows/netuio/resource.h

This file is not needed.

Quoted 6 lines
windows/netuio/kernel/README_NetUIO.rst new file +0 -0
new file mode 100644
index 000000000..a290fcf20
@@ -0,0 +1,64 @@

[snip]

windows/netuio/kernel/README_NetUIO.rst line 48

Quoted 6 lines
windows/netuio/kernel/README_NetUIO.rst new file +6 -0
+Installing netuio.sys Driver for development
+--------------------------------------------
+.. note::
+
+In a development environment, NetUIO driver can be loaded by enabling test-signing.
+Please implement adequate precautionary measures before installing a test-signed driver, replacing a signed-driver.

Paragraph under "note" must be indented for RST to peek it up as note content. Like so:

.. note::

This line has 3 spaces in the beginning.

[snip]

windows/netuio/kernel/windows/netuio/netuio.inf line 2

Quoted 8 lines
windows/netuio/kernel/windows/netuio/netuio.inf new file +2 -0
new file mode 100644
index 000000000..88e90b365
@@ -0,0 +1,78 @@
+; SPDX-License-Identifier: BSD-3-Clause
+; Copyright (c) Microsoft Corporation. All rights reserved

This copyright differs from the one in README.
"All rights reserved" is probably unnecessary.

[snip]

windows/netuio/kernel/windows/netuio/netuio.inf line 71

Quoted 4 lines
windows/netuio/kernel/windows/netuio/netuio.inf new file +4 -0
+[Strings]
+SPSVCINST_ASSOCSERVICE= 0x00000002
+Intel = "Intel"
+Broadcom = "Broadcom Corporation"

IHVs are supposed to add this gradually.

Quoted 12 lines
windows/netuio/kernel/windows/netuio/netuio.inf new file +7 -0
+ClassName = "Windows UIO"
+DiskName = "DPDK netUIO Installation Disk"
+netuio.DeviceDesc = "netuio Device"
+netuio.SVCDESC = "netuio Service"
+
+[DMAr.reg]
+HKR,Parameters,DmaRemappingCompatible,0x00010001,1
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +0 -0
new file mode 100644
index 000000000..6394bb5d1

[snip]

Quoted 3 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +3 -0
+NTSTATUS
+netuio_create_device(_Inout_ PWDFDEVICE_INIT DeviceInit)
+{

[snip]

windows/netuio/kernel/windows/netuio/netuio_dev.c line 57

Quoted 7 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +7 -0
+
+    status = WdfDeviceCreate(&DeviceInit, &deviceAttributes, &device);
+
+	if (NT_SUCCESS(status)) {
+		// Create a device interface so that applications can find and talk to us.
+		status = WdfDeviceCreateDeviceInterface(device, &GUID_DEVINTERFACE_netUIO, NULL);
+	}

Mixed tabs and spaces for indent.

[snip]

windows/netuio/kernel/windows/netuio/netuio_dev.c line 84

Quoted 3 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +3 -0
+/*
+Routine Description:
+    Free all the resources allocated in AdfEvtDeviceAdd.

Typo: Adf -> Wdf.

[snip]

Quoted 3 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +3 -0
+static NTSTATUS
+get_pci_device_info(_In_ WDFOBJECT device)
+{

[snip]

windows/netuio/kernel/windows/netuio/netuio_dev.c line 273

Quoted 6 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +6 -0
+        // Also, retrieve the NUMA node of the device
+        USHORT numaNode;
+        status = IoGetDeviceNumaNode(pdo, &numaNode);
+        if (NT_SUCCESS(status)) {
+            netuio_contextdata->dev_numa_node = numaNode;
+        }

Why is this needed? Userspace has access to this info.

Quoted 19 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +19 -0
+    }
+
+    return status;
+}
+
+static NTSTATUS
+create_device_specific_symbolic_link(_In_ WDFOBJECT device)
+{
+    NTSTATUS status = STATUS_UNSUCCESSFUL;
+    UNICODE_STRING netuio_symbolic_link;
+
+    PNETUIO_CONTEXT_DATA  netuio_contextdata;
+    netuio_contextdata = netuio_get_context_data(device);
+
+    if (!netuio_contextdata)
+        return status;
+
+    // Build symbolic link name as <netuio_symbolic_link>_BDF  (bus/device/func)
+    CHAR  symbolic_link[64] = { 0 };

windows/netuio/kernel/windows/netuio/netuio_dev.c line 302

Quoted 10 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +10 -0
+    sprintf_s(symbolic_link, sizeof(symbolic_link), "%s_%04d%02d%02d",
+                            NETUIO_DEVICE_SYMBOLIC_LINK_ANSI, netuio_contextdata->addr.bus_num,
+                            netuio_contextdata->addr.dev_num, netuio_contextdata->addr.func_num);
+
+    ANSI_STRING ansi_symbolic_link;
+    RtlInitAnsiString(&ansi_symbolic_link, symbolic_link);
+
+    status = RtlAnsiStringToUnicodeString(&netuio_symbolic_link, &ansi_symbolic_link, TRUE);
+    if (!NT_SUCCESS(status))
+        return status;

Why not use Unicode directly?

Quoted 7 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +7 -0
+
+    status = WdfDeviceCreateSymbolicLink(device, &netuio_symbolic_link);
+
+    RtlFreeUnicodeString(&netuio_symbolic_link);
+
+    return status;
+}

[snip]

windows/netuio/kernel/windows/netuio/netuio_dev.c

Quoted 6 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +6 -0
+
+// Local function protoyypes
+static NTSTATUS get_pci_device_info(_In_ WDFOBJECT device);
+static NTSTATUS create_device_specific_symbolic_link(_In_ WDFOBJECT device);
+static NTSTATUS allocate_usermemory_segment(_In_ WDFOBJECT device);
+static VOID free_usermemory_segment(_In_ WDFOBJECT device);

Nit: local functions usually don't appear in headers.

[snip]

Quoted 19 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +13 -0
new file mode 100644
index 000000000..6674931d2
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) Microsoft Corporation. All rights reserved
+ */
+
+#ifndef NETUIO_INTERFACE_H
+#define NETUIO_INTERFACE_H
+
+// All structures in this file are packed on an 8B boundary. 
+#pragma pack(push)
+#pragma pack(8)
+
+// Define an Interface Guid so that any app can find the device and talk to it.
+DEFINE_GUID (GUID_DEVINTERFACE_netUIO, 0x08336f60,0x0679,0x4c6c,0x85,0xd2,0xae,0x7c,0xed,0x65,0xff,0xf7); // {08336f60-0679-4c6c-85d2-ae7ced65fff7}

windows/netuio/kernel/windows/netuio/netuio_interface.h line 23

Quoted 10 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +10 -0
+
+// Device name definitions
+#define NETUIO_DEVICE_SYMBOLIC_LINK_ANSI    "\\DosDevices\\netuio"
+
+// netUIO driver symbolic name (prefix)
+#define NETUIO_DRIVER_NAME  _T("netuio")
+
+// IOCTL code definitions
+#define IOCTL_NETUIO_MAP_HW_INTO_USERMODE CTL_CODE(FILE_DEVICE_NETWORK, 51, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)
+#define IOCTL_NETUIO_PCI_CONFIG_IO        CTL_CODE(FILE_DEVICE_NETWORK, 52, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)

Input and output of these interfaces has to be documented somewhere.

Quoted 12 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +12 -0
+
+struct mem_region {
+    UINT64           size;       // memory region size
+    PHYSICAL_ADDRESS phys_addr;  // physical address of the memory region
+    PVOID            virt_addr;  // virtual address of the memory region
+    PVOID            user_mapped_virt_addr;  // virtual address of the region mapped into user process context
+};
+
+struct dev_addr {
+    ULONG   bus_num;
+    USHORT  dev_num;
+    USHORT  func_num;

windows/netuio/kernel/windows/netuio/netuio_interface.h line 45

Quoted 10 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +10 -0
+};
+
+enum pci_io {
+    PCI_IO_READ = 0,
+    PCI_IO_WRITE = 1
+};
+
+#define PCI_MAX_BAR 6
+
+struct dpdk_private_info

Nit: "private" usually means something hidden, not exposed like this. How about "device_info", "device_data", or "device_resources"?

windows/netuio/kernel/windows/netuio/netuio_interface.h line 49

Quoted 4 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +4 -0
+{
+    struct mem_region   hw[PCI_MAX_BAR];
+    struct mem_region   ms;
+    struct dev_addr     dev_addr;

Why is the address needed? Each device has its own symlink and handle. Such interface is harder to use with no visible benefit. Same below.

windows/netuio/kernel/windows/netuio/netuio_interface.h line 60

Quoted 11 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +11 -0
+    UINT16              dev_id;
+    UINT16              sub_dev_id;
+    USHORT              dev_numa_node;
+    USHORT              reserved;
+};
+
+struct dpdk_pci_config_io
+{
+    struct dev_addr     dev_addr;
+    UINT32              offset;
+    enum pci_io         op;

Driver API should use fixed-size types. Enum size is unspecified.

Quoted 13 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +13 -0
+    UINT32              access_size; // 1, 2, 4, or 8 bytes
+
+    union dpdk_pci_config_io_data {
+        UINT8			u8;
+        UINT16			u16;
+        UINT32			u32;
+        UINT64			u64;
+    } data;
+};
+
+#pragma pack(pop)
+
+#endif // NETUIO_INTERFACE_H

[snip]

windows/netuio/kernel/windows/netuio/netuio_interface.h

Quoted 11 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +11 -0
+static NTSTATUS
+netuio_map_address_into_user_process(_In_ PNETUIO_CONTEXT_DATA netuio_contextdata)
+{
+    NTSTATUS status = STATUS_SUCCESS;
+
+    // Map the scratch memory regions to the user's process context
+    MmBuildMdlForNonPagedPool(netuio_contextdata->dpdk_seg.mdl);
+    __try {
+        netuio_contextdata->dpdk_seg.mem.user_mapped_virt_addr =
+            MmMapLockedPagesSpecifyCache(netuio_contextdata->dpdk_seg.mdl, UserMode,
+                                         MmCached, NULL, FALSE, NormalPagePriority);

If user does IOCTL twice, they'll get the same memory mapped twice to different addresses, but the driver will only remember the last mapping. After it is removed, the first mapping will remain, exposing kernel memory for user-after-free from userspace. Or do I miss something here?

DPDK doesn't need multiple mappings, so you could either return an error or just an already mapped address on the second call.

[snip]

Quoted 6 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +6 -0
+NTSTATUS
+netuio_queue_initialize(_In_ WDFDEVICE Device)
+{
+    WDFQUEUE queue;
+    NTSTATUS status;
+    WDF_IO_QUEUE_CONFIG    queueConfig;

windows/netuio/kernel/windows/netuio/netuio_interface.h

Quoted 10 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +10 -0
+
+    PAGED_CODE();
+
+    // Configure a default queue so that requests that are not
+    // configure-fowarded using WdfDeviceConfigureRequestDispatching to goto
+    // other queues get dispatched here.
+    WDF_IO_QUEUE_CONFIG_INIT_DEFAULT_QUEUE(&queueConfig, WdfIoQueueDispatchParallel);
+
+    queueConfig.EvtIoDeviceControl = netuio_evt_IO_device_control;
+    queueConfig.EvtIoStop = netuio_evt_IO_stop;

EvtIoStop is not needed.

[snip]

Quoted 19 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +19 -0
+_Use_decl_annotations_
+VOID
+netuio_evt_IO_in_caller_context(
+    IN WDFDEVICE  Device,
+    IN WDFREQUEST Request
+)
+{
+    WDF_REQUEST_PARAMETERS params = { 0 };
+    NTSTATUS status = STATUS_SUCCESS;
+    PVOID    input_buf = NULL, output_buf = NULL;
+    size_t   input_buf_size, output_buf_size;
+    size_t  bytes_returned = 0;
+    PNETUIO_CONTEXT_DATA  netuio_contextdata = NULL;
+
+    netuio_contextdata = netuio_get_context_data(Device);
+
+    WDF_REQUEST_PARAMETERS_INIT(&params);
+    WdfRequestGetParameters(Request, &params);
+

windows/netuio/kernel/windows/netuio/netuio_interface.h

Quoted 10 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +10 -0
+    // We only need to be in the context of the process that initiated the request
+    //when we need to map memory to userspace. Otherwise, send the request back to framework.
+    if (!((params.Type == WdfRequestTypeDeviceControl) &&
+        (params.Parameters.DeviceIoControl.IoControlCode == IOCTL_NETUIO_MAP_HW_INTO_USERMODE)))
+    {
+        status = WdfDeviceEnqueueRequest(Device, Request);
+
+        if (!NT_SUCCESS(status))
+        {
+            WdfRequestCompleteWithInformation(Request, status, bytes_returned);

Nit: WdfRequestComplete(Request, status) would be sufficient.

Quoted 4 lines
windows/netuio/kernel/windows/netuio/netuio_interface.h new file +4 -0
+        }
+        return;
+    }
+

[snip]

Quoted 6 lines
windows/netuio/kernel/windows/netuio/netuio_queue.h new file +0 -0
new file mode 100644
index 000000000..9173a77b9

windows/netuio/kernel/windows/netuio/netuio_queue.h line 8

Quoted 8 lines
windows/netuio/kernel/windows/netuio/netuio_queue.h new file +8 -0
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) Microsoft Corporation. All rights reserved
+ */
+
+#ifndef NETUIO_QUEUE_H
+#define NETUIO_QUEUE_H
+
+EXTERN_C_START

Not needed in C code.

windows/netuio/kernel/windows/netuio/netuio_queue.h line 15

Quoted 7 lines
windows/netuio/kernel/windows/netuio/netuio_queue.h new file +7 -0
+
+// This is the context that can be placed per queue and would contain per queue information.
+typedef struct _QUEUE_CONTEXT {
+    ULONG PrivateDeviceData;  // just a placeholder
+} QUEUE_CONTEXT, *PQUEUE_CONTEXT;
+
+WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(QUEUE_CONTEXT, QueueGetContext)

Boilerplate, not needed.

[snip]

#6 lore raw

PATCH v3 1/1 windows/netuio: add Windows NetUIO kernel driver

In reply to Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERMODE
- used for mapping the device registers into userspace

Description of output:
- the physical address, virtual address and the size of the
memory region where the BARs were mapped.

* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Description of input:
- the operation type (read/write)
- the offset into the device data where the operation begins
- the length of data to read/write.
- in case of a write operation, the data to be written to
the device configuration space.
Description of output:
- in case of a read operation, the output buffer is filled
with the data read from the configuration space.

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>

---

v3:
* Remove memory segment allocation and mapping logic,
since DPDK handles memory management
* Remove the PCI address from IOCTL calls
* Add check to verify if device registers are already mapped
to usermode to avoid overwriting previous mappings
* Change enum to UINT8
* Remove dev_id and dev_numa_node from the interface
* Move source files and Visual Studio files to windows\netuio
* Remove resource.h and netuio.vcxproj.user as they get generated by VS
* Rename the README, add reference to the README in the parent folder, minor fixes
* Place _Use_decl_annotations_ on function definitions
* Move git files inside windows folder
* Various cleanups: renaming variables, removing dead code, fixing license text

 windows/.gitattributes                |   4 +
 windows/.gitignore                    |   2 +
 windows/netuio/README.rst             |  58 +++++
 windows/netuio/netuio.inf             |  77 ++++++
 windows/netuio/netuio.sln             |  24 ++
 windows/netuio/netuio.vcxproj         | 113 +++++++++
 windows/netuio/netuio.vcxproj.filters |  54 +++++
 windows/netuio/netuio_dev.c           | 286 ++++++++++++++++++++++
 windows/netuio/netuio_dev.h           |  50 ++++
 windows/netuio/netuio_drv.c           | 158 ++++++++++++
 windows/netuio/netuio_drv.h           |  27 +++
 windows/netuio/netuio_interface.h     |  67 ++++++
 windows/netuio/netuio_queue.c         | 333 ++++++++++++++++++++++++++
 windows/netuio/netuio_queue.h         |  19 ++
 14 files changed, 1272 insertions(+)
 create mode 100644 windows/.gitattributes
 create mode 100644 windows/.gitignore
 create mode 100644 windows/netuio/README.rst
 create mode 100644 windows/netuio/netuio.inf
 create mode 100644 windows/netuio/netuio.sln
 create mode 100644 windows/netuio/netuio.vcxproj
 create mode 100644 windows/netuio/netuio.vcxproj.filters
 create mode 100644 windows/netuio/netuio_dev.c
 create mode 100644 windows/netuio/netuio_dev.h
 create mode 100644 windows/netuio/netuio_drv.c
 create mode 100644 windows/netuio/netuio_drv.h
 create mode 100644 windows/netuio/netuio_interface.h
 create mode 100644 windows/netuio/netuio_queue.c
 create mode 100644 windows/netuio/netuio_queue.h
windows/.gitattributes new file +4 -0
new file mode 100644
index 000000000..13482db3d
@@ -0,0 +1,4 @@
+* text=auto
+
+*.sln text eol=crlf
+*.vcxproj text eol=crlf
windows/.gitignore new file +2 -0
new file mode 100644
index 000000000..543281e8f
@@ -0,0 +1,2 @@
+x64/
+.vs/
windows/netuio/README.rst new file +58 -0

58 lines not on this page · raw message

windows/netuio/netuio.inf new file +77 -0

77 lines not on this page · raw message

windows/netuio/netuio.sln new file +24 -0
new file mode 100644
index 000000000..15c26e6b9
@@ -0,0 +1,24 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25420.1
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "netuio", "netuio.vcxproj", "{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.ActiveCfg = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Build.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Deploy.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.ActiveCfg = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Build.0 = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Deploy.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
windows/netuio/netuio.vcxproj new file +113 -0

114 lines not on this page · raw message

windows/netuio/netuio.vcxproj.filters new file +54 -0

55 lines not on this page · raw message

windows/netuio/netuio_dev.c new file +286 -0

286 lines not on this page · raw message

windows/netuio/netuio_dev.h new file +50 -0

50 lines not on this page · raw message

windows/netuio/netuio_drv.c new file +158 -0

158 lines not on this page · raw message

windows/netuio/netuio_drv.h new file +27 -0
new file mode 100644
index 000000000..5bc0f90f9
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#ifndef NETUIO_DRV_H
+#define NETUIO_DRV_H
+
+#define INITGUID
+
+#include <ntddk.h>
+#include <wdf.h>
+
+#include "netuio_dev.h"
+#include "netuio_queue.h"
+
+// Print output constants
+#define DPFLTR_NETUIO_INFO_LEVEL   35
+
+// WDFDRIVER Events
+DRIVER_INITIALIZE DriverEntry;
+EVT_WDF_DRIVER_DEVICE_ADD       netuio_evt_device_add;
+EVT_WDF_OBJECT_CONTEXT_CLEANUP  netuio_evt_driver_context_cleanup;
+EVT_WDF_DEVICE_PREPARE_HARDWARE netuio_evt_prepare_hw;
+EVT_WDF_DEVICE_RELEASE_HARDWARE netuio_evt_release_hw;
+EVT_WDF_FILE_CLOSE              netuio_evt_file_cleanup;
+
+#endif // NETUIO_DRV_H
windows/netuio/netuio_interface.h new file +67 -0

67 lines not on this page · raw message

windows/netuio/netuio_queue.c new file +333 -0

333 lines not on this page · raw message

windows/netuio/netuio_queue.h new file +19 -0
new file mode 100644
index 000000000..42f75be38
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#ifndef NETUIO_QUEUE_H
+#define NETUIO_QUEUE_H
+
+VOID
+netuio_unmap_address_from_user_process(_In_ PNETUIO_CONTEXT_DATA netuio_contextdata);
+
+NTSTATUS
+netuio_queue_initialize(_In_ WDFDEVICE hDevice);
+
+// Events from the IoQueue object
+EVT_WDF_IO_QUEUE_IO_DEVICE_CONTROL netuio_evt_IO_device_control;
+
+EVT_WDF_IO_IN_CALLER_CONTEXT netuio_evt_IO_in_caller_context;
+
+#endif // NETUIO_QUEUE_H
Signature · 1 line
2.23.0.vfs.1.1.63.g5a5ad7f
#4 lore raw
In reply to Dmitry Kozlyuk
Quoted 29 lines · Dmitry Kozlyuk
Quoted 9 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows NetUIO kernel driver allows the DPDK userspace application to directly access the hardware.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>

---

Major questions:

1. Does NetUIO still need to allocate and map a contiguous memory segment now, when DPDK has user-mode memory management?

2. IOCTLs require to specify PCI address on each call. This is very inconvenient for DPDK consumers and also seems to serve no purpose.

3. There is a need to document driver's design, preferably in commit message, specifically:

3.1) DMA remapping capability in INF (AFAIK, vendors are notified); 3.2) manual BAR probing instead of using resource lists; 3.3) reason to use EvtIoInCallerContext and IO queues; 3.4) IOCTL format.

Also, I agree with everything Ranjit has noted already.

Quoted 9 lines

General suggestions to cleanup the code a bit. We can do it later if you wish. This also brings up the question, which code style should Windows kernel code for DPDK follow (off-topic for now).

* Remove boilerplate code and comments generated by VS wizard.
* Place _Use_decl_annotations on definitions to make them simpler.
* Limit line length, try using shorted variable names (e.g. "netuio_contextdata" may be "context" or "ctx" with no loss).

Thank you for you thorough review!

Quoted 10 lines

More specific comments inline.

[snip]

windows/netuio/kernel/windows/netuio/netuio.inf line 71

Quoted 4 lines

+[Strings]

windows/netuio/kernel/windows/netuio/netuio.inf new file +3 -0
+SPSVCINST_ASSOCSERVICE= 0x00000002
+Intel = "Intel"
+Broadcom = "Broadcom Corporation"

IHVs are supposed to add this gradually.

Thanks! I've modified the inf to remove all the vendor specific strings but left the sections to avoid errors when generating the binary.

Quoted 11 lines
Quoted 11 lines
(quoted diff) +11 -0
+ClassName = "Windows UIO"
+DiskName = "DPDK netUIO Installation Disk"
+netuio.DeviceDesc = "netuio Device"
+netuio.SVCDESC = "netuio Service"
+
+    // Build symbolic link name as <netuio_symbolic_link>_BDF  (bus/device/func)
+    CHAR  symbolic_link[64] = { 0 };
+    sprintf_s(symbolic_link, sizeof(symbolic_link), "%s_%04d%02d%02d",
+                            NETUIO_DEVICE_SYMBOLIC_LINK_ANSI, netuio_contextdata->addr.bus_num,
+                            netuio_contextdata->addr.dev_num, netuio_contextdata->addr.func_num);
+
Quoted 9 lines

windows/netuio/kernel/windows/netuio/netuio_dev.c line 302

Quoted 6 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +6 -0
+    ANSI_STRING ansi_symbolic_link;
+    RtlInitAnsiString(&ansi_symbolic_link, symbolic_link);
+
+    status = RtlAnsiStringToUnicodeString(&netuio_symbolic_link, &ansi_symbolic_link, TRUE);
+    if (!NT_SUCCESS(status))
+        return status;

Why not use Unicode directly?

It looks like either way, a cast will be needed (for example, if I use wchar for symbolic_link and then RtlInitUnicodeString()). I've left it as is, but let me know if there's an elegant solution that I didn't see.

Quoted 6 lines
Quoted 6 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +6 -0
+
+    status = WdfDeviceCreateSymbolicLink(device, &netuio_symbolic_link);
+
+    RtlFreeUnicodeString(&netuio_symbolic_link);
+
+    return status;
#2 lore raw
In reply to Ranjit Menon
Quoted 33 lines · Ranjit Menon

Hi, Naty

Quoted 10 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows NetUIO kernel driver allows the DPDK userspace application to directly access the hardware.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>

---

I haven't reviewed the code yet, but a few general suggestions and observations:

1. This code is going into the dpdk-kmods repo, so we don't need the four-level deep directory path for the source.

2. The code can simply be in the windows/netuio directory (don't need 'kernel')

3. The 'virt2phys' module that is already in the repo puts the build-related files (project files, solution files etc) in the same folder as the source files. We should do the same here. We don't need the mk/exec-env subfolder for these files. They can be in the same directory as the source.

4. Remove the .vcxproj.user file from the patch. It is not required.

5. The README should be renamed to simply README.rst and it can also be in this windows/netuio folder

Quoted 9 lines

6. The existing README.rst in the windows folder contains information on how to build and load kernel modules in Windows. This new README can reference the root-level README, if required. (Also some of the information in this new README is out-of-date. I'll review that separately)

ranjit m.

Thank you, Ranjit! I've cleaned it up and sent v3.

#7 lore raw
In reply to Narcisa Ana Maria Vasile
Quoted 1 line · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

Thanks for the hard work! A few comments inline worth checking, but this version improved much anyway.

Quoted 16 lines

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERMODE
- used for mapping the device registers into userspace

Description of output:
- the physical address, virtual address and the size of the
memory region where the BARs were mapped.

* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Quoted 9 lines

Description of input:
- the operation type (read/write)
- the offset into the device data where the operation begins
- the length of data to read/write.
- in case of a write operation, the data to be written to
the device configuration space.
Description of output:
- in case of a read operation, the output buffer is filled
with the data read from the configuration space.

Detailed parameter descriptions would better suit netuio_interface.h, where the IOCTLs are defined. Short ones are useful here, though.

Quoted 1 line
 windows/netuio/README.rst             |  58 +++++

Nit: "git am" complains about spaces at end of a few lines in this file.

[snip]

Quoted 3 lines
windows/netuio/netuio_dev.c new file +3 -0
+static NTSTATUS
+get_pci_device_info(_In_ WDFOBJECT device)
+{

[snip]

windows/netuio/netuio_dev.c line 41

Quoted 6 lines
windows/netuio/netuio_dev.c new file +6 -0
+    // Retrieve the B:D:F details of our device
+    PDEVICE_OBJECT pdo = NULL;
+    pdo = WdfDeviceWdmGetPhysicalDevice(device);
+    if (pdo) {
+        ULONG prop = 0, length = 0;
+        status = IoGetDeviceProperty(pdo, DevicePropertyBusNumber, sizeof(ULONG), (PVOID)&ctx->addr.bus_num, &length);

Status is not checked and is immediately overwritten.

Quoted 12 lines
windows/netuio/netuio_dev.c new file +12 -0
+        status = IoGetDeviceProperty(pdo, DevicePropertyAddress, sizeof(ULONG), (PVOID)&prop, &length);
+
+        if (NT_SUCCESS(status)) {
+            ctx->addr.func_num = prop & 0x0000FFFF;
+            ctx->addr.dev_num = ((prop >> 16) & 0x0000FFFF);
+        }
+    }
+
+    return status;
+}
+
+static NTSTATUS

windows/netuio/netuio_dev.c line 63

Quoted 10 lines
windows/netuio/netuio_dev.c new file +10 -0
+create_device_specific_symbolic_link(_In_ WDFOBJECT device)
+{
+    NTSTATUS status = STATUS_UNSUCCESSFUL;
+    UNICODE_STRING netuio_symbolic_link;
+
+    PNETUIO_CONTEXT_DATA  ctx;
+    ctx = netuio_get_context_data(device);
+
+    if (!ctx)
+        return status;

Can "ctx" really be NULL? Same applies to file object context. If I get WdfObjectGetTypedContext() description right, it cannot, so you could save a lot of checks.

windows/netuio/netuio_dev.c line 69

Quoted 6 lines
windows/netuio/netuio_dev.c new file +6 -0
+
+    // Build symbolic link name as <netuio_symbolic_link>_BDF  (bus/device/func)
+    CHAR  symbolic_link[64] = { 0 };
+    sprintf_s(symbolic_link, sizeof(symbolic_link), "%s_%04d%02d%02d",
+        NETUIO_DEVICE_SYMBOLIC_LINK_ANSI, ctx->addr.bus_num,
+        ctx->addr.dev_num, ctx->addr.func_num);

You probably want to use %x, and not %d for PCI addresses, because decimal device numbers may not fit 2 digits and it will be hard to recognize. The other place is DbgPrintEx() below. No strong opinion, however, because Device Manager seems to use decimal.

Quoted 14 lines
windows/netuio/netuio_dev.c new file +14 -0
+
+    ANSI_STRING ansi_symbolic_link;
+    RtlInitAnsiString(&ansi_symbolic_link, symbolic_link);
+
+    status = RtlAnsiStringToUnicodeString(&netuio_symbolic_link, &ansi_symbolic_link, TRUE);
+    if (!NT_SUCCESS(status))
+        return status;
+
+    status = WdfDeviceCreateSymbolicLink(device, &netuio_symbolic_link);
+
+    RtlFreeUnicodeString(&netuio_symbolic_link);
+
+    return status;
+}

[snip]

windows/netuio/netuio_dev.c line 148

Quoted 5 lines
windows/netuio/netuio_dev.c new file +5 -0
+
+_Use_decl_annotations_
+NTSTATUS
+netuio_map_hw_resources(WDFDEVICE Device, WDFCMRESLIST Resources, WDFCMRESLIST ResourcesTranslated)
+{

Could you please document the solution here? Something along these lines:

ResourcesTranslated report MMIO BARs in the correct order, but their indices differ from physical ones. Traverse PCI configuration manually to maintain physical index, searching for the next MMIO resource each time.

When I first implemented a netuio-like driver, I fell into this trap with indices. Worse, I did work sometimes, e.g. with virtio-pci (legacy),that's why I consider this non-obvious.

[snip]

Quoted 4 lines
windows/netuio/netuio_dev.c new file +4 -0
+    for (INT bar_index = 0; bar_index < PCI_MAX_BAR; bar_index++) {
+        prev_bar = curr_bar;
+        curr_bar = pci_config.u.type0.BaseAddresses[bar_index];
+        if (curr_bar == 0 || (prev_bar & PCI_TYPE_64BIT)) {

windows/netuio/netuio_dev.c line 198

Quoted 10 lines
windows/netuio/netuio_dev.c new file +10 -0
+            // Skip this bar
+            ctx->bar[bar_index].base_addr.QuadPart = 0;
+            ctx->bar[bar_index].size = 0;
+            ctx->bar[bar_index].virt_addr = 0;
+
+            ctx->dpdk_hw[bar_index].mdl = NULL;
+            ctx->dpdk_hw[bar_index].mem.size = 0;
+
+            continue;
+        }

Maybe RtlZeroMmeory() the whole array beforehand and just continue here? This would protect from forgetting to clear new fields, should they be added.

Quoted 4 lines
windows/netuio/netuio_dev.c new file +4 -0
+
+        // Find next CmResourceTypeMemory
+        do {
+            descriptor = WdfCmResourceListGetDescriptor(ResourcesTranslated, next_descriptor);

windows/netuio/netuio_dev.c line 212

Quoted 10 lines
windows/netuio/netuio_dev.c new file +10 -0
+            next_descriptor++;
+
+            if (descriptor == NULL) {
+                status = STATUS_DEVICE_CONFIGURATION_ERROR;
+                goto end;
+            }
+        } while (descriptor->Type != CmResourceTypeMemory);
+
+        // Assert that we have the correct descriptor
+        ASSERT((descriptor->Flags & CM_RESOURCE_MEMORY_BAR) != 0);

Shouldn't this assert be a check? Or incorporated in loop condition maybe.

windows/netuio/netuio_dev.c line 223

Quoted 11 lines
windows/netuio/netuio_dev.c new file +11 -0
+
+        if (curr_bar & PCI_TYPE_64BIT) {
+            ASSERT(bar_index != PCI_TYPE0_ADDRESSES - 1);
+            bar_addr = ((ULONGLONG)pci_config.u.type0.BaseAddresses[bar_index + 1] << 32) | (curr_bar & PCI_ADDRESS_MEMORY_ADDRESS_MASK);
+        }
+        else
+        {
+            bar_addr = curr_bar & PCI_ADDRESS_MEMORY_ADDRESS_MASK;
+        }
+
+        ASSERT((ULONGLONG)descriptor->u.Memory.Start.QuadPart == bar_addr);

Are these calculations really needed if you end up using address from the resource descriptor?

Quoted 6 lines
windows/netuio/netuio_drv.c new file +6 -0
+
+/*
+Routine Description:
+    Free all the resources allocated in DriverEntry.
+
+Return Value:

windows/netuio/netuio_drv.c line 124

Quoted 10 lines
windows/netuio/netuio_drv.c new file +10 -0
+    None
+-*/
+_Use_decl_annotations_
+VOID
+netuio_evt_driver_context_cleanup(WDFOBJECT DriverObject)
+{
+    UNREFERENCED_PARAMETER(DriverObject);
+    DbgPrintEx(DPFLTR_IHVNETWORK_ID, DPFLTR_NETUIO_INFO_LEVEL, "netUIO Driver unloaded.\n");
+    PAGED_CODE();
+}

The comment is misleading, no resources are freed here. You could've done without this function, driver loading/unloading is logged anyway. Also, PAGED_CODE() logically must come before any IRQL-constrained calls.

[snip]

Quoted 16 lines
windows/netuio/netuio_drv.h new file +1 -0
+#endif // NETUIO_DRV_H
windows/netuio/netuio_interface.h new file +9 -0
new file mode 100644
index 000000000..6fc75ad8e
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#ifndef NETUIO_INTERFACE_H
+#define NETUIO_INTERFACE_H
+
+// All structures in this file are packed on an 8B boundary. 
+#pragma pack(push)

windows/netuio/netuio_interface.h line 19

Quoted 10 lines
windows/netuio/netuio_interface.h new file +10 -0
+#pragma pack(8)
+
+// Define an Interface Guid so that any app can find the device and talk to it.
+DEFINE_GUID (GUID_DEVINTERFACE_netUIO, 0x08336f60,0x0679,0x4c6c,0x85,0xd2,0xae,0x7c,0xed,0x65,0xff,0xf7); // {08336f60-0679-4c6c-85d2-ae7ced65fff7}
+
+// Device name definitions
+#define NETUIO_DEVICE_SYMBOLIC_LINK_ANSI    "\\DosDevices\\netuio"
+
+// netUIO driver symbolic name (prefix)
+#define NETUIO_DRIVER_NAME  _T("netuio")

This string is unused and is not needed for interface.

Quoted 7 lines
windows/netuio/netuio_interface.h new file +7 -0
+
+// IOCTL code definitions
+#define IOCTL_NETUIO_MAP_HW_INTO_USERMODE CTL_CODE(FILE_DEVICE_NETWORK, 51, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)
+#define IOCTL_NETUIO_PCI_CONFIG_IO        CTL_CODE(FILE_DEVICE_NETWORK, 52, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)
+
+struct mem_region {
+    UINT64           size;       // memory region size

windows/netuio/netuio_interface.h line 36

Quoted 10 lines
windows/netuio/netuio_interface.h new file +10 -0
+    LARGE_INTEGER    phys_addr;  // physical address of the memory region
+    PVOID            virt_addr;  // virtual address of the memory region
+    PVOID            user_mapped_virt_addr;  // virtual address of the region mapped into user process context
+};
+
+struct dev_addr {
+    ULONG   bus_num;
+    USHORT  dev_num;
+    USHORT  func_num;
+};

This structure is not needed for interface.

windows/netuio/netuio_interface.h line 48

Quoted 12 lines
windows/netuio/netuio_interface.h new file +12 -0
+
+enum pci_io {
+    PCI_IO_READ = 0,
+    PCI_IO_WRITE = 1
+};
+
+#define PCI_MAX_BAR 6
+
+struct device_info
+{
+    struct mem_region   hw[PCI_MAX_BAR];
+    USHORT              reserved;

"Reserved" is not used and it is not documented whether the caller must initialize it or not. I suggest removing it.

Quoted 39 lines
windows/netuio/netuio_interface.h new file +19 -0
+};
+
+struct dpdk_pci_config_io
+{
+    UINT32              offset;
+    UINT8               op;
+    UINT32              access_size; // 1, 2, 4, or 8 bytes
+
+    union dpdk_pci_config_io_data {
+        UINT8			u8;
+        UINT16			u16;
+        UINT32			u32;
+        UINT64			u64;
+    } data;
+};
+
+#pragma pack(pop)
+
+#endif // NETUIO_INTERFACE_H
windows/netuio/netuio_queue.c new file +14 -0
new file mode 100644
index 000000000..cea94e630
@@ -0,0 +1,333 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#include "netuio_drv.h"
+
+#ifdef ALLOC_PRAGMA
+#pragma alloc_text (PAGE, netuio_queue_initialize)
+#endif
+
+static
+BOOLEAN
+netuio_get_usermode_mapping_flag(WDFREQUEST Request)
+{

[snip]

Quoted 1 line

+}

Blank line is missing here.

[snip]

Quoted 14 lines
windows/netuio/netuio_queue.c new file +14 -0
+
+/*
+Routine Description:
+    Maps address ranges into the usermode process's address space.  The following
+    ranges are mapped:
+
+        * Any PCI BARs that our device was assigned
+        * The scratch buffer of contiguous pages
+
+Return Value:
+    NTSTATUS
+*/
+static NTSTATUS
+netuio_map_address_into_user_process(_In_ PNETUIO_CONTEXT_DATA ctx, WDFREQUEST Request)

[snip]

Quoted 25 lines
windows/netuio/netuio_queue.c new file +25 -0
+_Use_decl_annotations_
+VOID
+netuio_evt_IO_in_caller_context(WDFDEVICE  Device,
+                                WDFREQUEST Request)
+{
+    WDF_REQUEST_PARAMETERS params = { 0 };
+    NTSTATUS status = STATUS_SUCCESS;
+    PVOID    output_buf = NULL;
+    size_t   output_buf_size;
+    size_t  bytes_returned = 0;
+    PNETUIO_CONTEXT_DATA  ctx = NULL;
+
+    ctx = netuio_get_context_data(Device);
+
+    WDF_REQUEST_PARAMETERS_INIT(&params);
+    WdfRequestGetParameters(Request, &params);
+
+    // We only need to be in the context of the process that initiated the request
+    //when we need to map memory to userspace. Otherwise, send the request back to framework.
+    if (!((params.Type == WdfRequestTypeDeviceControl) &&
+        (params.Parameters.DeviceIoControl.IoControlCode == IOCTL_NETUIO_MAP_HW_INTO_USERMODE)))
+    {
+        status = WdfDeviceEnqueueRequest(Device, Request);
+
+        if (!NT_SUCCESS(status))

windows/netuio/netuio_queue.c line 228

Quoted 10 lines
windows/netuio/netuio_queue.c new file +10 -0
+        {
+            WdfRequestCompleteWithInformation(Request, status, bytes_returned);
+        }
+        return;
+    }
+
+    // Return relevant data to the caller
+    status = WdfRequestRetrieveOutputBuffer(Request, sizeof(struct device_info), &output_buf, &output_buf_size);
+    if (!NT_SUCCESS(status)) {
+        status = STATUS_INVALID_BUFFER_SIZE;

Better propagate original error to help debugging (two more similar places in netuio_evt_IO_device_control).

Quoted 31 lines
windows/netuio/netuio_queue.c new file +31 -0
+        goto end;
+    }
+
+    status = netuio_map_address_into_user_process(ctx, Request);
+    if (status != STATUS_SUCCESS) {
+        goto end;
+    }
+
+    netuio_set_usermode_mapping_flag(Request);
+
+    netuio_handle_get_hw_data_request(ctx, output_buf, output_buf_size);
+    bytes_returned = output_buf_size;
+
+end:
+    WdfRequestCompleteWithInformation(Request, status, bytes_returned);
+
+    return;
+}
+
+/*
+Routine Description:
+    This event is invoked when the framework receives IRP_MJ_DEVICE_CONTROL request.
+
+Return Value:
+    None
+ */
+_Use_decl_annotations_
+VOID
+netuio_evt_IO_device_control(WDFQUEUE Queue, WDFREQUEST Request,
+                             size_t OutputBufferLength, size_t InputBufferLength,
+                             ULONG IoControlCode)

[snip]

Quoted 8 lines
windows/netuio/netuio_queue.c new file +8 -0
+
+    if (dpdk_pci_io_input->op == PCI_IO_READ) {
+        *(UINT64 *)output_buf = 0;
+        bytes_returned = ctx->bus_interface.GetBusData(
+            ctx->bus_interface.Context,
+            PCI_WHICHSPACE_CONFIG,
+            output_buf,
+            dpdk_pci_io_input->offset,

windows/netuio/netuio_queue.c line 321

Quoted 10 lines
windows/netuio/netuio_queue.c new file +10 -0
+            dpdk_pci_io_input->access_size);
+    }
+    else if (dpdk_pci_io_input->op == PCI_IO_WRITE) {
+        // returns bytes written
+        bytes_returned = ctx->bus_interface.SetBusData(
+            ctx->bus_interface.Context,
+            PCI_WHICHSPACE_CONFIG,
+            (PVOID)&dpdk_pci_io_input->data,
+            dpdk_pci_io_input->offset,
+            dpdk_pci_io_input->access_size);

The "offset" is not checked, so GetBusData() and SetBusData() can fail and return 0. You have to either check it above to ensure success or to check if bytes_returned != dpdk_pci_io_input->access_size and complete with error.

Quoted 11 lines
windows/netuio/netuio_queue.c new file +11 -0
+    }
+    else {
+        status = STATUS_INVALID_PARAMETER;
+        goto end;
+    }
+
+end:
+    WdfRequestCompleteWithInformation(Request, status, bytes_returned);
+
+    return;
+}
#5 lore raw
In reply to Narcisa Ana Maria Vasile
Quoted 2 lines · Narcisa Ana Maria Vasile
Quoted 1 line · Dmitry Kozlyuk

On Thu, 20 Aug 2020 15:23:55 -0700, Narcisa Ana Maria Vasile wrote:

[snip]

Quoted 20 lines
Quoted 11 lines
Quoted 11 lines
(quoted diff) +11 -0
+ClassName = "Windows UIO"
+DiskName = "DPDK netUIO Installation Disk"
+netuio.DeviceDesc = "netuio Device"
+netuio.SVCDESC = "netuio Service"
+
+    // Build symbolic link name as <netuio_symbolic_link>_BDF  (bus/device/func)
+    CHAR  symbolic_link[64] = { 0 };
+    sprintf_s(symbolic_link, sizeof(symbolic_link), "%s_%04d%02d%02d",
+                            NETUIO_DEVICE_SYMBOLIC_LINK_ANSI, netuio_contextdata->addr.bus_num,
+                            netuio_contextdata->addr.dev_num, netuio_contextdata->addr.func_num);
+
Quoted 9 lines

windows/netuio/kernel/windows/netuio/netuio_dev.c line 302

Quoted 6 lines
windows/netuio/kernel/windows/netuio/netuio_dev.c new file +6 -0
+    ANSI_STRING ansi_symbolic_link;
+    RtlInitAnsiString(&ansi_symbolic_link, symbolic_link);
+
+    status = RtlAnsiStringToUnicodeString(&netuio_symbolic_link, &ansi_symbolic_link, TRUE);
+    if (!NT_SUCCESS(status))
+        return status;  

Why not use Unicode directly?

Quoted 2 lines

It looks like either way, a cast will be needed (for example, if I use wchar for symbolic_link and then RtlInitUnicodeString()). I've left it as is, but let me know if there's an elegant solution that I didn't see.

It's not that important, really, you can leave the code as-is. FWIW, I meant something like this:

DECLARE_UNICODE_STRING_SIZE(symbolic_link_name, NETUIO_MAX_SYMLINK_LEN); RtlUnicodeStringPrintf(

    symbolic_link_name,
    NETUIO_DEVICE_SYMBOLIC_LINK_UNICODE,
    ...);

https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/ntstrsafe/nf-ntstrsafe-rtlunicodestringprintf

#8 lore raw

PATCH v4 1/1 windows/netuio: add Windows NetUIO kernel driver

In reply to Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERMODE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>

---

v4:
Fixed issues reported by DmitryK:
* Fix whitespaces issues
* Fix status not checked
* Remove NULL checks for device and file context getters
* Use %x for BDF
* Add comment for BAR traversing
* Remove complex calculations that were only needed in ASSERT
* Remove netuio_evt_driver_context_cleanup as not needed
* Check GetBusData/SetBusData returned bytes
* General cleanup

 windows/.gitattributes                |   4 +
 windows/.gitignore                    |   2 +
 windows/netuio/README.rst             |  58 +++++
 windows/netuio/netuio.inf             |  77 ++++++
 windows/netuio/netuio.sln             |  24 ++
 windows/netuio/netuio.vcxproj         | 113 +++++++++
 windows/netuio/netuio.vcxproj.filters |  54 +++++
 windows/netuio/netuio_dev.c           | 273 +++++++++++++++++++++
 windows/netuio/netuio_dev.h           |  66 +++++
 windows/netuio/netuio_drv.c           | 131 ++++++++++
 windows/netuio/netuio_drv.h           |  30 +++
 windows/netuio/netuio_interface.h     |  88 +++++++
 windows/netuio/netuio_queue.c         | 334 ++++++++++++++++++++++++++
 windows/netuio/netuio_queue.h         |  21 ++
 14 files changed, 1275 insertions(+)
 create mode 100644 windows/.gitattributes
 create mode 100644 windows/.gitignore
 create mode 100644 windows/netuio/README.rst
 create mode 100644 windows/netuio/netuio.inf
 create mode 100644 windows/netuio/netuio.sln
 create mode 100644 windows/netuio/netuio.vcxproj
 create mode 100644 windows/netuio/netuio.vcxproj.filters
 create mode 100644 windows/netuio/netuio_dev.c
 create mode 100644 windows/netuio/netuio_dev.h
 create mode 100644 windows/netuio/netuio_drv.c
 create mode 100644 windows/netuio/netuio_drv.h
 create mode 100644 windows/netuio/netuio_interface.h
 create mode 100644 windows/netuio/netuio_queue.c
 create mode 100644 windows/netuio/netuio_queue.h
windows/.gitattributes new file +4 -0
new file mode 100644
index 000000000..13482db3d
@@ -0,0 +1,4 @@
+* text=auto
+
+*.sln text eol=crlf
+*.vcxproj text eol=crlf
windows/.gitignore new file +2 -0
new file mode 100644
index 000000000..543281e8f
@@ -0,0 +1,2 @@
+x64/
+.vs/
windows/netuio/README.rst new file +58 -0

58 lines not on this page · raw message

windows/netuio/netuio.inf new file +77 -0

77 lines not on this page · raw message

windows/netuio/netuio.sln new file +24 -0
new file mode 100644
index 000000000..15c26e6b9
@@ -0,0 +1,24 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25420.1
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "netuio", "netuio.vcxproj", "{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.ActiveCfg = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Build.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Deploy.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.ActiveCfg = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Build.0 = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Deploy.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
windows/netuio/netuio.vcxproj new file +113 -0

114 lines not on this page · raw message

windows/netuio/netuio.vcxproj.filters new file +54 -0

55 lines not on this page · raw message

windows/netuio/netuio_dev.c new file +273 -0

273 lines not on this page · raw message

windows/netuio/netuio_dev.h new file +66 -0

66 lines not on this page · raw message

windows/netuio/netuio_drv.c new file +131 -0

131 lines not on this page · raw message

windows/netuio/netuio_drv.h new file +30 -0
new file mode 100644
index 000000000..469c6ac60
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#ifndef NETUIO_DRV_H
+#define NETUIO_DRV_H
+
+#define INITGUID
+
+#include <ntddk.h>
+#include <wdf.h>
+
+#include "netuio_dev.h"
+#include "netuio_queue.h"
+
+/**
+ * Print output constants
+ */
+#define DPFLTR_NETUIO_INFO_LEVEL   35
+
+/**
+ * WDFDRIVER Events
+ */
+DRIVER_INITIALIZE DriverEntry;
+EVT_WDF_DRIVER_DEVICE_ADD       netuio_evt_device_add;
+EVT_WDF_DEVICE_PREPARE_HARDWARE netuio_evt_prepare_hw;
+EVT_WDF_DEVICE_RELEASE_HARDWARE netuio_evt_release_hw;
+EVT_WDF_FILE_CLOSE              netuio_evt_file_cleanup;
+
+#endif // NETUIO_DRV_H
windows/netuio/netuio_interface.h new file +88 -0

88 lines not on this page · raw message

windows/netuio/netuio_queue.c new file +334 -0

334 lines not on this page · raw message

windows/netuio/netuio_queue.h new file +21 -0
new file mode 100644
index 000000000..29888537c
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#ifndef NETUIO_QUEUE_H
+#define NETUIO_QUEUE_H
+
+VOID
+netuio_unmap_address_from_user_process(_In_ PNETUIO_CONTEXT_DATA netuio_contextdata);
+
+NTSTATUS
+netuio_queue_initialize(_In_ WDFDEVICE hDevice);
+
+/**
+ * Events from the IoQueue object
+ */
+EVT_WDF_IO_QUEUE_IO_DEVICE_CONTROL netuio_evt_IO_device_control;
+
+EVT_WDF_IO_IN_CALLER_CONTEXT netuio_evt_IO_in_caller_context;
+
+#endif // NETUIO_QUEUE_H
Signature · 1 line
2.23.0.vfs.1.1.63.g5a5ad7f
Reviewed-by on v4 1/1
#9 lore raw
Quoted 23 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERMODE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Quoted 5 lines
Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>

---

#10 lore raw
In reply to Narcisa Ana Maria Vasile
Quoted 70 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERMODE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>

---

v4:
Fixed issues reported by DmitryK:
* Fix whitespaces issues
* Fix status not checked
* Remove NULL checks for device and file context getters
* Use %x for BDF
* Add comment for BAR traversing
* Remove complex calculations that were only needed in ASSERT
* Remove netuio_evt_driver_context_cleanup as not needed
* Check GetBusData/SetBusData returned bytes
* General cleanup

  windows/.gitattributes                |   4 +
  windows/.gitignore                    |   2 +
  windows/netuio/README.rst             |  58 +++++
  windows/netuio/netuio.inf             |  77 ++++++
  windows/netuio/netuio.sln             |  24 ++
  windows/netuio/netuio.vcxproj         | 113 +++++++++
  windows/netuio/netuio.vcxproj.filters |  54 +++++
  windows/netuio/netuio_dev.c           | 273 +++++++++++++++++++++
  windows/netuio/netuio_dev.h           |  66 +++++
  windows/netuio/netuio_drv.c           | 131 ++++++++++
  windows/netuio/netuio_drv.h           |  30 +++
  windows/netuio/netuio_interface.h     |  88 +++++++
  windows/netuio/netuio_queue.c         | 334 ++++++++++++++++++++++++++
  windows/netuio/netuio_queue.h         |  21 ++
  14 files changed, 1275 insertions(+)

create mode 100644 windows/.gitattributes create mode 100644 windows/.gitignore
create mode 100644 windows/netuio/README.rst create mode 100644 windows/netuio/netuio.inf create mode 100644 windows/netuio/netuio.sln create mode 100644 windows/netuio/netuio.vcxproj create mode 100644 windows/netuio/netuio.vcxproj.filters create mode 100644 windows/netuio/netuio_dev.c create mode 100644 windows/netuio/netuio_dev.h create mode 100644 windows/netuio/netuio_drv.c create mode 100644 windows/netuio/netuio_drv.h create mode 100644 windows/netuio/netuio_interface.h create mode 100644 windows/netuio/netuio_queue.c create mode 100644 windows/netuio/netuio_queue.h

<snip>

Quoted 30 lines
windows/netuio/netuio_interface.h new file +24 -0
new file mode 100644
index 000000000..232b5bc2d
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+ /**
+  * @file netuio kernel driver interface
+  */
+
+#ifndef NETUIO_INTERFACE_H
+#define NETUIO_INTERFACE_H
+
+/**
+ * All structures in this file are packed on an 8B boundary.
+ */
+#pragma pack(push)
+#pragma pack(8)
+
+/**
+ * Define an Interface Guid so that any app can find the device and talk to it.
+ */
+DEFINE_GUID (GUID_DEVINTERFACE_netUIO, 0x08336f60,0x0679,0x4c6c,0x85,0xd2,0xae,0x7c,0xed,0x65,0xff,0xf7); // {08336f60-0679-4c6c-85d2-ae7ced65fff7}
+
+/**
+ * Device name definitions

windows/netuio/netuio_interface.h line 34

Quoted 10 lines
windows/netuio/netuio_interface.h new file +10 -0
+ */
+#define NETUIO_DEVICE_SYMBOLIC_LINK_UNICODE    L"\\DosDevices\\netuio"
+#define NETUIO_MAX_SYMLINK_LEN                 255
+
+/**
+ * IOCTL_NETUIO_MAP_HW_INTO_USERMODE is used for mapping the device registers
+ * into userspace. It returns the physical address, virtual address
+ * and the size of the memory region where the BARs were mapped.
+ */
+#define IOCTL_NETUIO_MAP_HW_INTO_USERMODE CTL_CODE(FILE_DEVICE_NETWORK, 51, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS)

Not critical, but can we rename this IOCTL code to

IOCTL_NETUIO_MAP_HW_INTO_USERSPACE since that seems more appropriate and it is what the comment is referring to?

<snip>

ranjit m.

#11 lore raw

PATCH v5 1/1 windows/netuio: add Windows NetUIO kernel driver

In reply to Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERSPACE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>

---

v5:
* Changed the name of the IOCTL for clarity

 windows/.gitattributes                |   4 +
 windows/.gitignore                    |   2 +
 windows/netuio/README.rst             |  58 +++++
 windows/netuio/netuio.inf             |  77 ++++++
 windows/netuio/netuio.sln             |  24 ++
 windows/netuio/netuio.vcxproj         | 113 +++++++++
 windows/netuio/netuio.vcxproj.filters |  54 +++++
 windows/netuio/netuio_dev.c           | 273 +++++++++++++++++++++
 windows/netuio/netuio_dev.h           |  66 +++++
 windows/netuio/netuio_drv.c           | 131 ++++++++++
 windows/netuio/netuio_drv.h           |  30 +++
 windows/netuio/netuio_interface.h     |  88 +++++++
 windows/netuio/netuio_queue.c         | 334 ++++++++++++++++++++++++++
 windows/netuio/netuio_queue.h         |  21 ++
 14 files changed, 1275 insertions(+)
 create mode 100644 windows/.gitattributes
 create mode 100644 windows/.gitignore
 create mode 100644 windows/netuio/README.rst
 create mode 100644 windows/netuio/netuio.inf
 create mode 100644 windows/netuio/netuio.sln
 create mode 100644 windows/netuio/netuio.vcxproj
 create mode 100644 windows/netuio/netuio.vcxproj.filters
 create mode 100644 windows/netuio/netuio_dev.c
 create mode 100644 windows/netuio/netuio_dev.h
 create mode 100644 windows/netuio/netuio_drv.c
 create mode 100644 windows/netuio/netuio_drv.h
 create mode 100644 windows/netuio/netuio_interface.h
 create mode 100644 windows/netuio/netuio_queue.c
 create mode 100644 windows/netuio/netuio_queue.h
windows/.gitattributes new file +4 -0
new file mode 100644
index 000000000..13482db3d
@@ -0,0 +1,4 @@
+* text=auto
+
+*.sln text eol=crlf
+*.vcxproj text eol=crlf
windows/.gitignore new file +2 -0
new file mode 100644
index 000000000..543281e8f
@@ -0,0 +1,2 @@
+x64/
+.vs/
windows/netuio/README.rst new file +58 -0

58 lines not on this page · raw message

windows/netuio/netuio.inf new file +77 -0

77 lines not on this page · raw message

windows/netuio/netuio.sln new file +24 -0
new file mode 100644
index 000000000..15c26e6b9
@@ -0,0 +1,24 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25420.1
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "netuio", "netuio.vcxproj", "{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.ActiveCfg = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Build.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Debug|x64.Deploy.0 = Debug|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.ActiveCfg = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Build.0 = Release|x64
+		{66EC91EF-AC5B-4D1E-8314-9B3E2855CCF6}.Release|x64.Deploy.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
windows/netuio/netuio.vcxproj new file +113 -0

114 lines not on this page · raw message

windows/netuio/netuio.vcxproj.filters new file +54 -0

55 lines not on this page · raw message

windows/netuio/netuio_dev.c new file +273 -0

273 lines not on this page · raw message

windows/netuio/netuio_dev.h new file +66 -0

66 lines not on this page · raw message

windows/netuio/netuio_drv.c new file +131 -0

131 lines not on this page · raw message

windows/netuio/netuio_drv.h new file +30 -0
new file mode 100644
index 000000000..469c6ac60
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#ifndef NETUIO_DRV_H
+#define NETUIO_DRV_H
+
+#define INITGUID
+
+#include <ntddk.h>
+#include <wdf.h>
+
+#include "netuio_dev.h"
+#include "netuio_queue.h"
+
+/**
+ * Print output constants
+ */
+#define DPFLTR_NETUIO_INFO_LEVEL   35
+
+/**
+ * WDFDRIVER Events
+ */
+DRIVER_INITIALIZE DriverEntry;
+EVT_WDF_DRIVER_DEVICE_ADD       netuio_evt_device_add;
+EVT_WDF_DEVICE_PREPARE_HARDWARE netuio_evt_prepare_hw;
+EVT_WDF_DEVICE_RELEASE_HARDWARE netuio_evt_release_hw;
+EVT_WDF_FILE_CLOSE              netuio_evt_file_cleanup;
+
+#endif // NETUIO_DRV_H
windows/netuio/netuio_interface.h new file +88 -0

88 lines not on this page · raw message

windows/netuio/netuio_queue.c new file +334 -0

334 lines not on this page · raw message

windows/netuio/netuio_queue.h new file +21 -0
new file mode 100644
index 000000000..29888537c
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Microsoft Corporation.
+ */
+
+#ifndef NETUIO_QUEUE_H
+#define NETUIO_QUEUE_H
+
+VOID
+netuio_unmap_address_from_user_process(_In_ PNETUIO_CONTEXT_DATA netuio_contextdata);
+
+NTSTATUS
+netuio_queue_initialize(_In_ WDFDEVICE hDevice);
+
+/**
+ * Events from the IoQueue object
+ */
+EVT_WDF_IO_QUEUE_IO_DEVICE_CONTROL netuio_evt_IO_device_control;
+
+EVT_WDF_IO_IN_CALLER_CONTEXT netuio_evt_IO_in_caller_context;
+
+#endif // NETUIO_QUEUE_H
Signature · 1 line
2.23.0.vfs.1.1.63.g5a5ad7f
Reviewed-by Acked-by on v5 1/1
#12 lore raw
Quoted 47 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERSPACE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>

---

v5:
* Changed the name of the IOCTL for clarity

  windows/.gitattributes                |   4 +
  windows/.gitignore                    |   2 +
  windows/netuio/README.rst             |  58 +++++
  windows/netuio/netuio.inf             |  77 ++++++
  windows/netuio/netuio.sln             |  24 ++
  windows/netuio/netuio.vcxproj         | 113 +++++++++
  windows/netuio/netuio.vcxproj.filters |  54 +++++
  windows/netuio/netuio_dev.c           | 273 +++++++++++++++++++++
  windows/netuio/netuio_dev.h           |  66 +++++
  windows/netuio/netuio_drv.c           | 131 ++++++++++
  windows/netuio/netuio_drv.h           |  30 +++
  windows/netuio/netuio_interface.h     |  88 +++++++
  windows/netuio/netuio_queue.c         | 334 ++++++++++++++++++++++++++
  windows/netuio/netuio_queue.h         |  21 ++
  14 files changed, 1275 insertions(+)
Quoted 14 lines

create mode 100644 windows/.gitattributes create mode 100644 windows/.gitignore
create mode 100644 windows/netuio/README.rst create mode 100644 windows/netuio/netuio.inf create mode 100644 windows/netuio/netuio.sln create mode 100644 windows/netuio/netuio.vcxproj create mode 100644 windows/netuio/netuio.vcxproj.filters create mode 100644 windows/netuio/netuio_dev.c create mode 100644 windows/netuio/netuio_dev.h create mode 100644 windows/netuio/netuio_drv.c create mode 100644 windows/netuio/netuio_drv.h create mode 100644 windows/netuio/netuio_interface.h create mode 100644 windows/netuio/netuio_queue.c create mode 100644 windows/netuio/netuio_queue.h

We'll need to do a code-style cleanup at some point, but for now:

Reviewed-by on v5 1/1
#15 lore raw
In reply to Narcisa Ana Maria Vasile
Quoted 23 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERSPACE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Quoted 4 lines
Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>
#13 lore raw
In reply to Ranjit Menon
Quoted 25 lines · Ranjit Menon
Quoted 23 lines · Narcisa Ana Maria Vasile

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERSPACE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Quoted 9 lines
Quoted 6 lines
Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>

---

We'll need to do a code-style cleanup at some point, but for now:

Yes, agreed. I'll send a patch for style cleanup after merging this one. Thanks, Ranjit.

Quoted 3 lines
#14 lore raw
In reply to Ranjit Menon

02/10/2020 20:21, Ranjit Menon:

Quoted 29 lines · Narcisa Ana Maria Vasile
Quoted 23 lines

From: Narcisa Vasile <[email protected]>

The Windows netuio kernel driver provides the DPDK userspace application with direct access to hardware, by mapping the HW registers in userspace and allowing read/write operations from/to the device configuration space.

Two IOCTLs are defined by the netuio interface:
* IOCTL_NETUIO_MAP_HW_INTO_USERSPACE
- used for mapping the device registers into userspace
* IOCTL_NETUIO_PCI_CONFIG_IO
- used to read/write from/into the device configuration space

Note:
Requests to map the device BARs into userspace need to be processed in the thread context of the process that initiated the mapping request. Otherwise, the BARs might end up mapped into an arbitrary process address space.
EvtIoInCallerContext is used to ensure the requests are handled in the right user thread context. Other requests (PCI config) are sent back to the framework and processed by the EvtIoDeviceControl callback.

Quoted 4 lines
Cc Harini Ramakrishnan <[email protected]>
Cc Omar Cardona <[email protected]>
Cc Dmitry Malloy <[email protected]>
Quoted 4 lines

We'll need to do a code-style cleanup at some point, but for now:

For info, Reviewed-by is stronger because it means you carefully checked the code.
I drop Acked-by redundancy in this case.

With Reviewed-by: Dmitry Kozlyuk <[email protected]>

Applied, thanks

For next patches, I would prefer a prefix "kmods" in the subject:
--subject-prefix='kmods PATCH'