Skip to contentSkip to conversation

drm/amd/amdgpu: New debugfs interface for MMIO registers

PATCH In review Tom St Denis 4 participants 8 messages Aug 24, 2021

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amd/amdgpu: New debugfs interface for MMIO registers

This new debugfs interface uses an IOCTL interface in order to pass along state information like SRBM and GRBM bank switching. This new interface also allows a full 32-bit MMIO address range which the previous didn't. With this new design we have room to grow the flexibility of the file as need be.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177 ++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
 2 files changed, 209 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +177 -0
index 277128846dd1..ab2d92f84da5 100644
@@ -279,6 +279,173 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 	return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 }
 
+static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
+{
+	struct amdgpu_debugfs_regs2_data *rd;
+
+	rd = kzalloc(sizeof *rd, GFP_KERNEL);
+	if (!rd)
+		return -ENOMEM;
+	rd->adev = file_inode(file)->i_private;
+	file->private_data = rd;
+
+	return 0;
+}
+
+static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
+{
+	kfree(file->private_data);
+	return 0;
+}
+
+static int amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, int write_en)
+{
+	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+	struct amdgpu_device *adev = rd->adev;
+	int result = 0, r;
+	uint32_t value;
+
+	if (rd->state.offset & 0x3)
+		return -EINVAL;
+
+	if (rd->state.use_grbm) {
+		if (rd->state.grbm.se == 0x3FF)
+			rd->state.grbm.se = 0xFFFFFFFF;
+		if (rd->state.grbm.sh == 0x3FF)
+			rd->state.grbm.sh = 0xFFFFFFFF;
+		if (rd->state.grbm.instance == 0x3FF)
+			rd->state.grbm.instance = 0xFFFFFFFF;
+	}
+
+	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+	if (r < 0) {
+		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		return r;
+	}
+
+	r = amdgpu_virt_enable_access_debugfs(adev);
+	if (r < 0) {
+		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		return r;
+	}
+
+	if (rd->state.use_grbm) {
+		if ((rd->state.grbm.sh != 0xFFFFFFFF && rd->state.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
+		    (rd->state.grbm.se != 0xFFFFFFFF && rd->state.grbm.se >= adev->gfx.config.max_shader_engines)) {
+			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+			amdgpu_virt_disable_access_debugfs(adev);
+			return -EINVAL;
+		}
+		mutex_lock(&adev->grbm_idx_mutex);
+		amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se,
+								rd->state.grbm.sh,
+								rd->state.grbm.instance);
+	} else if (rd->state.use_grbm) {
+		mutex_lock(&adev->srbm_mutex);
+		amdgpu_gfx_select_me_pipe_q(adev, rd->state.srbm.me, rd->state.srbm.pipe,
+									rd->state.srbm.queue, rd->state.srbm.vmid);
+	}
+
+	if (rd->state.pg_lock)
+		mutex_lock(&adev->pm.mutex);
+
+	if (!write_en) {
+		value = RREG32(rd->state.offset >> 2);
+		r = put_user(value, (uint32_t *)buf);
+	} else {
+		r = get_user(value, (uint32_t *)buf);
+		if (!r)
+			amdgpu_mm_wreg_mmio_rlc(adev, rd->state.offset >> 2, value);
+	}
+	if (r) {
+		result = r;
+		goto end;
+	}
+	result = 0;
+end:
+	if (rd->state.use_grbm) {
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+		mutex_unlock(&adev->grbm_idx_mutex);
+	} else if (rd->state.use_srbm) {
+		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
+		mutex_unlock(&adev->srbm_mutex);
+	}
+
+	if (rd->state.pg_lock)
+		mutex_unlock(&adev->pm.mutex);
+
+	// in umr (the likely user of this) flags are set per file operation
+	// which means they're never "unset" explicitly.  To avoid breaking
+	// this convention we unset the flags after each operation
+	// flags are for a single call (need to be set for every read/write)
+	rd->state.use_grbm = 0;
+	rd->state.use_srbm = 0;
+	rd->state.pg_lock  = 0;
+
+	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+	amdgpu_virt_disable_access_debugfs(adev);
+	return result;
+}
+
+static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
+{
+	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+	unsigned char st[32], v;
+	int r, x;
+
+	// always read first 4 bytes of data
+	for (x = 0; x < 4; x++) {
+		if ((r = get_user(v, (unsigned char *)data))) {
+			return r;
+		}
+		++data;
+		st[x] = v;
+	}
+
+	// first 4 bytes are offset
+	rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
+					   ((u32)st[2] << 16) | ((u32)st[3] << 24);
+
+	switch (cmd) {
+		case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
+			for (x = 4; x < 32; x++) {
+				if ((r = get_user(v, (unsigned char *)data))) {
+					return r;
+				}
+				++data;
+				st[x] = v;
+			}
+
+			// next byte contains the flag
+			// we only consume the remainder of the state if bit 1 is set
+			// this allows the subsequent read/write
+			rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
+			rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
+			rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
+
+			// next 6 bytes are grbm data
+			rd->state.grbm.se       =  ((u32)st[5]) | ((u32)st[6] << 8);
+			rd->state.grbm.sh       =  ((u32)st[7]) | ((u32)st[8] << 8);
+			rd->state.grbm.instance =  ((u32)st[9]) | ((u32)st[10] << 8);
+
+			// next 8 are srbm data
+			rd->state.srbm.me       =  ((u32)st[11]) | ((u32)st[12] << 8);
+			rd->state.srbm.pipe     =  ((u32)st[13]) | ((u32)st[14] << 8);
+			rd->state.srbm.queue    =  ((u32)st[15]) | ((u32)st[16] << 8);
+			rd->state.srbm.vmid     =  ((u32)st[17]) | ((u32)st[18] << 8);
+			break;
+		case AMDGPU_DEBUGFS_REGS2_IOC_READ:
+			return amdgpu_debugfs_regs2_op(f, (char __user *)data, 0);
+		case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
+			return amdgpu_debugfs_regs2_op(f, (char __user *)data, 1);
+		default:
+			return -EINVAL;
+	}
+	return 0;
+}
 
 /**
  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
@@ -1091,6 +1258,14 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
 	return result;
 }
 
+static const struct file_operations amdgpu_debugfs_regs2_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
+	.open = amdgpu_debugfs_regs2_open,
+	.release = amdgpu_debugfs_regs2_release,
+	.llseek = default_llseek
+};
+
 static const struct file_operations amdgpu_debugfs_regs_fops = {
 	.owner = THIS_MODULE,
 	.read = amdgpu_debugfs_regs_read,
@@ -1148,6 +1323,7 @@ static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
 
 static const struct file_operations *debugfs_regs[] = {
 	&amdgpu_debugfs_regs_fops,
+	&amdgpu_debugfs_regs2_fops,
 	&amdgpu_debugfs_regs_didt_fops,
 	&amdgpu_debugfs_regs_pcie_fops,
 	&amdgpu_debugfs_regs_smc_fops,
@@ -1160,6 +1336,7 @@ static const struct file_operations *debugfs_regs[] = {
 
 static const char *debugfs_regs_names[] = {
 	"amdgpu_regs",
+	"amdgpu_regs2",
 	"amdgpu_regs_didt",
 	"amdgpu_regs_pcie",
 	"amdgpu_regs_smc",
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +32 -0
index 141a8474e24f..04c81cd4bcc7 100644
@@ -22,6 +22,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include <linux/ioctl.h>
 
 /*
  * Debugfs
@@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
 void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
 int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+
+struct amdgpu_debugfs_regs2_data {
+	struct amdgpu_device *adev;
+	struct {
+		// regs state
+		int use_srbm,
+		    use_grbm,
+		    pg_lock;
+		struct {
+			u32 se, sh, instance;
+		} grbm;
+		struct {
+			u32 me, pipe, queue, vmid;
+		} srbm;
+		u32 offset;
+	} state;
+};
+
+enum AMDGPU_DEBUGFS_REGS2_CMDS {
+	AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
+	AMDGPU_DEBUGFS_REGS2_CMD_READ,
+	AMDGPU_DEBUGFS_REGS2_CMD_WRITE,
+};
+
+struct amdgpu_debugfs_regs2_iocdata {
+	unsigned char t[32];
+};
+
+#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)
Signature · 1 line
2.31.1
#1 lore raw

Am 24.08.21 um 14:16 schrieb Tom St Denis:

Quoted 244 lines

This new debugfs interface uses an IOCTL interface in order to pass along state information like SRBM and GRBM bank switching. This new interface also allows a full 32-bit MMIO address range which the previous didn't. With this new design we have room to grow the flexibility of the file as need be.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177 ++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
  2 files changed, 209 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +177 -0
index 277128846dd1..ab2d92f84da5 100644
@@ -279,6 +279,173 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
  	return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
  }
 
+static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
+{
+	struct amdgpu_debugfs_regs2_data *rd;
+
+	rd = kzalloc(sizeof *rd, GFP_KERNEL);
+	if (!rd)
+		return -ENOMEM;
+	rd->adev = file_inode(file)->i_private;
+	file->private_data = rd;
+
+	return 0;
+}
+
+static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
+{
+	kfree(file->private_data);
+	return 0;
+}
+
+static int amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, int write_en)
+{
+	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+	struct amdgpu_device *adev = rd->adev;
+	int result = 0, r;
+	uint32_t value;
+
+	if (rd->state.offset & 0x3)
+		return -EINVAL;
+
+	if (rd->state.use_grbm) {
+		if (rd->state.grbm.se == 0x3FF)
+			rd->state.grbm.se = 0xFFFFFFFF;
+		if (rd->state.grbm.sh == 0x3FF)
+			rd->state.grbm.sh = 0xFFFFFFFF;
+		if (rd->state.grbm.instance == 0x3FF)
+			rd->state.grbm.instance = 0xFFFFFFFF;
+	}
+
+	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+	if (r < 0) {
+		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		return r;
+	}
+
+	r = amdgpu_virt_enable_access_debugfs(adev);
+	if (r < 0) {
+		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		return r;
+	}
+
+	if (rd->state.use_grbm) {
+		if ((rd->state.grbm.sh != 0xFFFFFFFF && rd->state.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
+		    (rd->state.grbm.se != 0xFFFFFFFF && rd->state.grbm.se >= adev->gfx.config.max_shader_engines)) {
+			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+			amdgpu_virt_disable_access_debugfs(adev);
+			return -EINVAL;
+		}
+		mutex_lock(&adev->grbm_idx_mutex);
+		amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se,
+								rd->state.grbm.sh,
+								rd->state.grbm.instance);
+	} else if (rd->state.use_grbm) {
+		mutex_lock(&adev->srbm_mutex);
+		amdgpu_gfx_select_me_pipe_q(adev, rd->state.srbm.me, rd->state.srbm.pipe,
+									rd->state.srbm.queue, rd->state.srbm.vmid);
+	}
+
+	if (rd->state.pg_lock)
+		mutex_lock(&adev->pm.mutex);
+
+	if (!write_en) {
+		value = RREG32(rd->state.offset >> 2);
+		r = put_user(value, (uint32_t *)buf);
+	} else {
+		r = get_user(value, (uint32_t *)buf);
+		if (!r)
+			amdgpu_mm_wreg_mmio_rlc(adev, rd->state.offset >> 2, value);
+	}
+	if (r) {
+		result = r;
+		goto end;
+	}
+	result = 0;
+end:
+	if (rd->state.use_grbm) {
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+		mutex_unlock(&adev->grbm_idx_mutex);
+	} else if (rd->state.use_srbm) {
+		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
+		mutex_unlock(&adev->srbm_mutex);
+	}
+
+	if (rd->state.pg_lock)
+		mutex_unlock(&adev->pm.mutex);
+
+	// in umr (the likely user of this) flags are set per file operation
+	// which means they're never "unset" explicitly.  To avoid breaking
+	// this convention we unset the flags after each operation
+	// flags are for a single call (need to be set for every read/write)
+	rd->state.use_grbm = 0;
+	rd->state.use_srbm = 0;
+	rd->state.pg_lock  = 0;
+
+	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+	amdgpu_virt_disable_access_debugfs(adev);
+	return result;
+}
+
+static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
+{
+	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+	unsigned char st[32], v;
+	int r, x;
+
+	// always read first 4 bytes of data
+	for (x = 0; x < 4; x++) {
+		if ((r = get_user(v, (unsigned char *)data))) {
+			return r;
+		}
+		++data;
+		st[x] = v;
+	}
+
+	// first 4 bytes are offset
+	rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
+					   ((u32)st[2] << 16) | ((u32)st[3] << 24);
+
+	switch (cmd) {
+		case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
+			for (x = 4; x < 32; x++) {
+				if ((r = get_user(v, (unsigned char *)data))) {
+					return r;
+				}
+				++data;
+				st[x] = v;
+			}
+
+			// next byte contains the flag
+			// we only consume the remainder of the state if bit 1 is set
+			// this allows the subsequent read/write
+			rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
+			rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
+			rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
+
+			// next 6 bytes are grbm data
+			rd->state.grbm.se       =  ((u32)st[5]) | ((u32)st[6] << 8);
+			rd->state.grbm.sh       =  ((u32)st[7]) | ((u32)st[8] << 8);
+			rd->state.grbm.instance =  ((u32)st[9]) | ((u32)st[10] << 8);
+
+			// next 8 are srbm data
+			rd->state.srbm.me       =  ((u32)st[11]) | ((u32)st[12] << 8);
+			rd->state.srbm.pipe     =  ((u32)st[13]) | ((u32)st[14] << 8);
+			rd->state.srbm.queue    =  ((u32)st[15]) | ((u32)st[16] << 8);
+			rd->state.srbm.vmid     =  ((u32)st[17]) | ((u32)st[18] << 8);
+			break;
+		case AMDGPU_DEBUGFS_REGS2_IOC_READ:
+			return amdgpu_debugfs_regs2_op(f, (char __user *)data, 0);
+		case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
+			return amdgpu_debugfs_regs2_op(f, (char __user *)data, 1);
+		default:
+			return -EINVAL;
+	}
+	return 0;
+}
 
  /**
   * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
@@ -1091,6 +1258,14 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
  	return result;
  }
 
+static const struct file_operations amdgpu_debugfs_regs2_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
+	.open = amdgpu_debugfs_regs2_open,
+	.release = amdgpu_debugfs_regs2_release,
+	.llseek = default_llseek
+};
+
  static const struct file_operations amdgpu_debugfs_regs_fops = {
  	.owner = THIS_MODULE,
  	.read = amdgpu_debugfs_regs_read,
@@ -1148,6 +1323,7 @@ static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
 
  static const struct file_operations *debugfs_regs[] = {
  	&amdgpu_debugfs_regs_fops,
+	&amdgpu_debugfs_regs2_fops,
  	&amdgpu_debugfs_regs_didt_fops,
  	&amdgpu_debugfs_regs_pcie_fops,
  	&amdgpu_debugfs_regs_smc_fops,
@@ -1160,6 +1336,7 @@ static const struct file_operations *debugfs_regs[] = {
 
  static const char *debugfs_regs_names[] = {
  	"amdgpu_regs",
+	"amdgpu_regs2",
  	"amdgpu_regs_didt",
  	"amdgpu_regs_pcie",
  	"amdgpu_regs_smc",
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +8 -0
index 141a8474e24f..04c81cd4bcc7 100644
@@ -22,6 +22,7 @@
   * OTHER DEALINGS IN THE SOFTWARE.
   *
   */
+#include <linux/ioctl.h>
 
  /*
   * Debugfs
@@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
  void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
  void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
  int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+
+struct amdgpu_debugfs_regs2_data {
+	struct amdgpu_device *adev;
+	struct {
+		// regs state
+		int use_srbm,
+		    use_grbm,

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 58

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +10 -0
+		    pg_lock;
+		struct {
+			u32 se, sh, instance;
+		} grbm;
+		struct {
+			u32 me, pipe, queue, vmid;
+		} srbm;
+		u32 offset;
+	} state;
+};

This stuff needs to be in some shared header then.

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 63

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +5 -0
+
+enum AMDGPU_DEBUGFS_REGS2_CMDS {
+	AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
+	AMDGPU_DEBUGFS_REGS2_CMD_READ,
+	AMDGPU_DEBUGFS_REGS2_CMD_WRITE,

Why not just using the normal read and write functions?

Christian.

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +9 -0
+};
+
+struct amdgpu_debugfs_regs2_iocdata {
+	unsigned char t[32];
+};
+
+#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)
#2 lore raw

[AMD Official Use Only]

What do you mean a "shared header?" How would they be shared between kernel and user?

As for why not read/write. Jus wanted to keep it simple. It's not really performance bound. umr never does reads/writes larger than 32-bits anyways. It doesn't have to be this way though but I figured the fewer LOC the better.

Tom

________________________________________
From: Christian König <[email protected]>
Sent: Tuesday, August 24, 2021 08:20
To: StDenis, Tom; [email protected]
Subject: Re: [PATCH] drm/amd/amdgpu: New debugfs interface for MMIO registers

Am 24.08.21 um 14:16 schrieb Tom St Denis:

Quoted 244 lines

This new debugfs interface uses an IOCTL interface in order to pass along state information like SRBM and GRBM bank switching. This new interface also allows a full 32-bit MMIO address range which the previous didn't. With this new design we have room to grow the flexibility of the file as need be.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177 ++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
  2 files changed, 209 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +177 -0
index 277128846dd1..ab2d92f84da5 100644
@@ -279,6 +279,173 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
      return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
  }
 
+static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
+{
+     struct amdgpu_debugfs_regs2_data *rd;
+
+     rd = kzalloc(sizeof *rd, GFP_KERNEL);
+     if (!rd)
+             return -ENOMEM;
+     rd->adev = file_inode(file)->i_private;
+     file->private_data = rd;
+
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
+{
+     kfree(file->private_data);
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, int write_en)
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     struct amdgpu_device *adev = rd->adev;
+     int result = 0, r;
+     uint32_t value;
+
+     if (rd->state.offset & 0x3)
+             return -EINVAL;
+
+     if (rd->state.use_grbm) {
+             if (rd->state.grbm.se == 0x3FF)
+                     rd->state.grbm.se = 0xFFFFFFFF;
+             if (rd->state.grbm.sh == 0x3FF)
+                     rd->state.grbm.sh = 0xFFFFFFFF;
+             if (rd->state.grbm.instance == 0x3FF)
+                     rd->state.grbm.instance = 0xFFFFFFFF;
+     }
+
+     r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     r = amdgpu_virt_enable_access_debugfs(adev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     if (rd->state.use_grbm) {
+             if ((rd->state.grbm.sh != 0xFFFFFFFF && rd->state.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
+                 (rd->state.grbm.se != 0xFFFFFFFF && rd->state.grbm.se >= adev->gfx.config.max_shader_engines)) {
+                     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+                     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+                     amdgpu_virt_disable_access_debugfs(adev);
+                     return -EINVAL;
+             }
+             mutex_lock(&adev->grbm_idx_mutex);
+             amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se,
+                                                             rd->state.grbm.sh,
+                                                             rd->state.grbm.instance);
+     } else if (rd->state.use_grbm) {
+             mutex_lock(&adev->srbm_mutex);
+             amdgpu_gfx_select_me_pipe_q(adev, rd->state.srbm.me, rd->state.srbm.pipe,
+                                                                     rd->state.srbm.queue, rd->state.srbm.vmid);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_lock(&adev->pm.mutex);
+
+     if (!write_en) {
+             value = RREG32(rd->state.offset >> 2);
+             r = put_user(value, (uint32_t *)buf);
+     } else {
+             r = get_user(value, (uint32_t *)buf);
+             if (!r)
+                     amdgpu_mm_wreg_mmio_rlc(adev, rd->state.offset >> 2, value);
+     }
+     if (r) {
+             result = r;
+             goto end;
+     }
+     result = 0;
+end:
+     if (rd->state.use_grbm) {
+             amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+             mutex_unlock(&adev->grbm_idx_mutex);
+     } else if (rd->state.use_srbm) {
+             amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
+             mutex_unlock(&adev->srbm_mutex);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_unlock(&adev->pm.mutex);
+
+     // in umr (the likely user of this) flags are set per file operation
+     // which means they're never "unset" explicitly.  To avoid breaking
+     // this convention we unset the flags after each operation
+     // flags are for a single call (need to be set for every read/write)
+     rd->state.use_grbm = 0;
+     rd->state.use_srbm = 0;
+     rd->state.pg_lock  = 0;
+
+     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+     amdgpu_virt_disable_access_debugfs(adev);
+     return result;
+}
+
+static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     unsigned char st[32], v;
+     int r, x;
+
+     // always read first 4 bytes of data
+     for (x = 0; x < 4; x++) {
+             if ((r = get_user(v, (unsigned char *)data))) {
+                     return r;
+             }
+             ++data;
+             st[x] = v;
+     }
+
+     // first 4 bytes are offset
+     rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
+                                        ((u32)st[2] << 16) | ((u32)st[3] << 24);
+
+     switch (cmd) {
+             case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
+                     for (x = 4; x < 32; x++) {
+                             if ((r = get_user(v, (unsigned char *)data))) {
+                                     return r;
+                             }
+                             ++data;
+                             st[x] = v;
+                     }
+
+                     // next byte contains the flag
+                     // we only consume the remainder of the state if bit 1 is set
+                     // this allows the subsequent read/write
+                     rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
+                     rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
+                     rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
+
+                     // next 6 bytes are grbm data
+                     rd->state.grbm.se       =  ((u32)st[5]) | ((u32)st[6] << 8);
+                     rd->state.grbm.sh       =  ((u32)st[7]) | ((u32)st[8] << 8);
+                     rd->state.grbm.instance =  ((u32)st[9]) | ((u32)st[10] << 8);
+
+                     // next 8 are srbm data
+                     rd->state.srbm.me       =  ((u32)st[11]) | ((u32)st[12] << 8);
+                     rd->state.srbm.pipe     =  ((u32)st[13]) | ((u32)st[14] << 8);
+                     rd->state.srbm.queue    =  ((u32)st[15]) | ((u32)st[16] << 8);
+                     rd->state.srbm.vmid     =  ((u32)st[17]) | ((u32)st[18] << 8);
+                     break;
+             case AMDGPU_DEBUGFS_REGS2_IOC_READ:
+                     return amdgpu_debugfs_regs2_op(f, (char __user *)data, 0);
+             case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
+                     return amdgpu_debugfs_regs2_op(f, (char __user *)data, 1);
+             default:
+                     return -EINVAL;
+     }
+     return 0;
+}
 
  /**
   * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
@@ -1091,6 +1258,14 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
      return result;
  }
 
+static const struct file_operations amdgpu_debugfs_regs2_fops = {
+     .owner = THIS_MODULE,
+     .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
+     .open = amdgpu_debugfs_regs2_open,
+     .release = amdgpu_debugfs_regs2_release,
+     .llseek = default_llseek
+};
+
  static const struct file_operations amdgpu_debugfs_regs_fops = {
      .owner = THIS_MODULE,
      .read = amdgpu_debugfs_regs_read,
@@ -1148,6 +1323,7 @@ static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
 
  static const struct file_operations *debugfs_regs[] = {
      &amdgpu_debugfs_regs_fops,
+     &amdgpu_debugfs_regs2_fops,
      &amdgpu_debugfs_regs_didt_fops,
      &amdgpu_debugfs_regs_pcie_fops,
      &amdgpu_debugfs_regs_smc_fops,
@@ -1160,6 +1336,7 @@ static const struct file_operations *debugfs_regs[] = {
 
  static const char *debugfs_regs_names[] = {
      "amdgpu_regs",
+     "amdgpu_regs2",
      "amdgpu_regs_didt",
      "amdgpu_regs_pcie",
      "amdgpu_regs_smc",
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +8 -0
index 141a8474e24f..04c81cd4bcc7 100644
@@ -22,6 +22,7 @@
   * OTHER DEALINGS IN THE SOFTWARE.
   *
   */
+#include <linux/ioctl.h>
 
  /*
   * Debugfs
@@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
  void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
  void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
  int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+
+struct amdgpu_debugfs_regs2_data {
+     struct amdgpu_device *adev;
+     struct {
+             // regs state
+             int use_srbm,
+                 use_grbm,

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 58

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +10 -0
+                 pg_lock;
+             struct {
+                     u32 se, sh, instance;
+             } grbm;
+             struct {
+                     u32 me, pipe, queue, vmid;
+             } srbm;
+             u32 offset;
+     } state;
+};

This stuff needs to be in some shared header then.

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 63

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +5 -0
+
+enum AMDGPU_DEBUGFS_REGS2_CMDS {
+     AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
+     AMDGPU_DEBUGFS_REGS2_CMD_READ,
+     AMDGPU_DEBUGFS_REGS2_CMD_WRITE,

Why not just using the normal read and write functions?

Christian.

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +9 -0
+};
+
+struct amdgpu_debugfs_regs2_iocdata {
+     unsigned char t[32];
+};
+
+#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)
#3 lore raw

Am 24.08.21 um 14:27 schrieb StDenis, Tom:

Quoted 3 lines

[AMD Official Use Only]

What do you mean a "shared header?" How would they be shared between kernel and user?

Somewhere in the include/uapi/drm/ folder I think. Either add that to amdgpu_drm.h or maybe amdgpu_debugfs.h?

Or just keep it as a structure in amdgpu_debugfs.h with a comment that it is used for an IOCTL.

Just not something hard coded like first byte is this, second that etc....

Quoted 1 line

As for why not read/write. Jus wanted to keep it simple. It's not really performance bound. umr never does reads/writes larger than 32-bits anyways. It doesn't have to be this way though but I figured the fewer LOC the better.

I think it would be cleaner if we would have separate functions for this.

As far as I can see you also don't need the dance with the power managerment etc for the IOCTL. It's just get_user() into your structure.

BTW: In the kernel coding style put "switch" and "case" on the same column, otherwise checkpatch.pl might complain.

Christian.

Quoted 286 lines

Tom

________________________________________
From: Christian König <[email protected]>
Sent: Tuesday, August 24, 2021 08:20
To: StDenis, Tom; [email protected]
Subject: Re: [PATCH] drm/amd/amdgpu: New debugfs interface for MMIO registers

Am 24.08.21 um 14:16 schrieb Tom St Denis:

Quoted 244 lines

This new debugfs interface uses an IOCTL interface in order to pass along state information like SRBM and GRBM bank switching. This new interface also allows a full 32-bit MMIO address range which the previous didn't. With this new design we have room to grow the flexibility of the file as need be.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177 ++++++++++++++++++++
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
   2 files changed, 209 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +177 -0
index 277128846dd1..ab2d92f84da5 100644
@@ -279,6 +279,173 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
       return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
   }
 
+static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
+{
+     struct amdgpu_debugfs_regs2_data *rd;
+
+     rd = kzalloc(sizeof *rd, GFP_KERNEL);
+     if (!rd)
+             return -ENOMEM;
+     rd->adev = file_inode(file)->i_private;
+     file->private_data = rd;
+
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
+{
+     kfree(file->private_data);
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, int write_en)
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     struct amdgpu_device *adev = rd->adev;
+     int result = 0, r;
+     uint32_t value;
+
+     if (rd->state.offset & 0x3)
+             return -EINVAL;
+
+     if (rd->state.use_grbm) {
+             if (rd->state.grbm.se == 0x3FF)
+                     rd->state.grbm.se = 0xFFFFFFFF;
+             if (rd->state.grbm.sh == 0x3FF)
+                     rd->state.grbm.sh = 0xFFFFFFFF;
+             if (rd->state.grbm.instance == 0x3FF)
+                     rd->state.grbm.instance = 0xFFFFFFFF;
+     }
+
+     r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     r = amdgpu_virt_enable_access_debugfs(adev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     if (rd->state.use_grbm) {
+             if ((rd->state.grbm.sh != 0xFFFFFFFF && rd->state.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
+                 (rd->state.grbm.se != 0xFFFFFFFF && rd->state.grbm.se >= adev->gfx.config.max_shader_engines)) {
+                     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+                     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+                     amdgpu_virt_disable_access_debugfs(adev);
+                     return -EINVAL;
+             }
+             mutex_lock(&adev->grbm_idx_mutex);
+             amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se,
+                                                             rd->state.grbm.sh,
+                                                             rd->state.grbm.instance);
+     } else if (rd->state.use_grbm) {
+             mutex_lock(&adev->srbm_mutex);
+             amdgpu_gfx_select_me_pipe_q(adev, rd->state.srbm.me, rd->state.srbm.pipe,
+                                                                     rd->state.srbm.queue, rd->state.srbm.vmid);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_lock(&adev->pm.mutex);
+
+     if (!write_en) {
+             value = RREG32(rd->state.offset >> 2);
+             r = put_user(value, (uint32_t *)buf);
+     } else {
+             r = get_user(value, (uint32_t *)buf);
+             if (!r)
+                     amdgpu_mm_wreg_mmio_rlc(adev, rd->state.offset >> 2, value);
+     }
+     if (r) {
+             result = r;
+             goto end;
+     }
+     result = 0;
+end:
+     if (rd->state.use_grbm) {
+             amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+             mutex_unlock(&adev->grbm_idx_mutex);
+     } else if (rd->state.use_srbm) {
+             amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
+             mutex_unlock(&adev->srbm_mutex);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_unlock(&adev->pm.mutex);
+
+     // in umr (the likely user of this) flags are set per file operation
+     // which means they're never "unset" explicitly.  To avoid breaking
+     // this convention we unset the flags after each operation
+     // flags are for a single call (need to be set for every read/write)
+     rd->state.use_grbm = 0;
+     rd->state.use_srbm = 0;
+     rd->state.pg_lock  = 0;
+
+     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+     amdgpu_virt_disable_access_debugfs(adev);
+     return result;
+}
+
+static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     unsigned char st[32], v;
+     int r, x;
+
+     // always read first 4 bytes of data
+     for (x = 0; x < 4; x++) {
+             if ((r = get_user(v, (unsigned char *)data))) {
+                     return r;
+             }
+             ++data;
+             st[x] = v;
+     }
+
+     // first 4 bytes are offset
+     rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
+                                        ((u32)st[2] << 16) | ((u32)st[3] << 24);
+
+     switch (cmd) {
+             case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
+                     for (x = 4; x < 32; x++) {
+                             if ((r = get_user(v, (unsigned char *)data))) {
+                                     return r;
+                             }
+                             ++data;
+                             st[x] = v;
+                     }
+
+                     // next byte contains the flag
+                     // we only consume the remainder of the state if bit 1 is set
+                     // this allows the subsequent read/write
+                     rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
+                     rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
+                     rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
+
+                     // next 6 bytes are grbm data
+                     rd->state.grbm.se       =  ((u32)st[5]) | ((u32)st[6] << 8);
+                     rd->state.grbm.sh       =  ((u32)st[7]) | ((u32)st[8] << 8);
+                     rd->state.grbm.instance =  ((u32)st[9]) | ((u32)st[10] << 8);
+
+                     // next 8 are srbm data
+                     rd->state.srbm.me       =  ((u32)st[11]) | ((u32)st[12] << 8);
+                     rd->state.srbm.pipe     =  ((u32)st[13]) | ((u32)st[14] << 8);
+                     rd->state.srbm.queue    =  ((u32)st[15]) | ((u32)st[16] << 8);
+                     rd->state.srbm.vmid     =  ((u32)st[17]) | ((u32)st[18] << 8);
+                     break;
+             case AMDGPU_DEBUGFS_REGS2_IOC_READ:
+                     return amdgpu_debugfs_regs2_op(f, (char __user *)data, 0);
+             case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
+                     return amdgpu_debugfs_regs2_op(f, (char __user *)data, 1);
+             default:
+                     return -EINVAL;
+     }
+     return 0;
+}
 
   /**
    * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
@@ -1091,6 +1258,14 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
       return result;
   }
 
+static const struct file_operations amdgpu_debugfs_regs2_fops = {
+     .owner = THIS_MODULE,
+     .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
+     .open = amdgpu_debugfs_regs2_open,
+     .release = amdgpu_debugfs_regs2_release,
+     .llseek = default_llseek
+};
+
   static const struct file_operations amdgpu_debugfs_regs_fops = {
       .owner = THIS_MODULE,
       .read = amdgpu_debugfs_regs_read,
@@ -1148,6 +1323,7 @@ static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
 
   static const struct file_operations *debugfs_regs[] = {
       &amdgpu_debugfs_regs_fops,
+     &amdgpu_debugfs_regs2_fops,
       &amdgpu_debugfs_regs_didt_fops,
       &amdgpu_debugfs_regs_pcie_fops,
       &amdgpu_debugfs_regs_smc_fops,
@@ -1160,6 +1336,7 @@ static const struct file_operations *debugfs_regs[] = {
 
   static const char *debugfs_regs_names[] = {
       "amdgpu_regs",
+     "amdgpu_regs2",
       "amdgpu_regs_didt",
       "amdgpu_regs_pcie",
       "amdgpu_regs_smc",
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +8 -0
index 141a8474e24f..04c81cd4bcc7 100644
@@ -22,6 +22,7 @@
    * OTHER DEALINGS IN THE SOFTWARE.
    *
    */
+#include <linux/ioctl.h>
 
   /*
    * Debugfs
@@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
   void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
   void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
   int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+
+struct amdgpu_debugfs_regs2_data {
+     struct amdgpu_device *adev;
+     struct {
+             // regs state
+             int use_srbm,
+                 use_grbm,

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 58

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +10 -0
+                 pg_lock;
+             struct {
+                     u32 se, sh, instance;
+             } grbm;
+             struct {
+                     u32 me, pipe, queue, vmid;
+             } srbm;
+             u32 offset;
+     } state;
+};

This stuff needs to be in some shared header then.

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 63

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +5 -0
+
+enum AMDGPU_DEBUGFS_REGS2_CMDS {
+     AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
+     AMDGPU_DEBUGFS_REGS2_CMD_READ,
+     AMDGPU_DEBUGFS_REGS2_CMD_WRITE,

Why not just using the normal read and write functions?

Christian.

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +9 -0
+};
+
+struct amdgpu_debugfs_regs2_iocdata {
+     unsigned char t[32];
+};
+
+#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)
#4 lore raw

The IOCTL data is in the debugfs header as it is. I could move that to the amdgpu_drm.h and include it from amdgpu_debugfs.h.

I'll re-write the STATE IOCTL to use a struct and then test against what I have in umr.

Refactoring the read/write is trivial and I'll do that no problem (with style fixes).

Cheers,
Tom

Quoted 358 lines · Christian König

Am 24.08.21 um 14:27 schrieb StDenis, Tom:

Quoted 3 lines

[AMD Official Use Only]

What do you mean a "shared header?" How would they be shared between

kernel and user?

Somewhere in the include/uapi/drm/ folder I think. Either add that to amdgpu_drm.h or maybe amdgpu_debugfs.h?

Or just keep it as a structure in amdgpu_debugfs.h with a comment that it is used for an IOCTL.

Just not something hard coded like first byte is this, second that etc....

Quoted 1 line

As for why not read/write. Jus wanted to keep it simple. It's not

really performance bound. umr never does reads/writes larger than 32-bits anyways. It doesn't have to be this way though but I figured the fewer LOC the better.

I think it would be cleaner if we would have separate functions for this.

As far as I can see you also don't need the dance with the power managerment etc for the IOCTL. It's just get_user() into your structure.

BTW: In the kernel coding style put "switch" and "case" on the same column, otherwise checkpatch.pl might complain.

Christian.

Quoted 8 lines

Tom

________________________________________
From: Christian König <[email protected]>
Sent: Tuesday, August 24, 2021 08:20
To: StDenis, Tom; [email protected]
Subject: Re: [PATCH] drm/amd/amdgpu: New debugfs interface for MMIO

registers

Quoted 13 lines

Am 24.08.21 um 14:16 schrieb Tom St Denis:

Quoted 9 lines

This new debugfs interface uses an IOCTL interface in order to pass along state information like SRBM and GRBM bank switching. This new interface also allows a full 32-bit MMIO address range which the previous didn't. With this new design we have room to grow the flexibility of the file as need be.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177

++++++++++++++++++++

Quoted 4 lines

Review

Quoted 4 lines
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
   2 files changed, 209 insertions(+)
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct

Quoted 4 lines

index 277128846dd1..ab2d92f84da5 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +0 -0
@@ -279,6 +279,173 @@ static ssize_t amdgpu_debugfs_regs_write(struct

file *f, const char __user *buf,

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 279

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +0 -0
       return amdgpu_debugfs_process_reg_op(false, f, (char __user

*)buf, size, pos);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 282

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
   }
 
+static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file

*file)

Quoted 5 lines
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +5 -0
+{
+     struct amdgpu_debugfs_regs2_data *rd;
+
+     rd = kzalloc(sizeof *rd, GFP_KERNEL);
+     if (!rd)
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+             return -ENOMEM;
+     rd->adev = file_inode(file)->i_private;
+     file->private_data = rd;
+
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_release(struct inode *inode, struct

file *file)

Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +6 -0
+{
+     kfree(file->private_data);
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_op(struct file *f, char __user *buf,

int write_en)

Quoted 33 lines
Quoted 22 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +22 -0
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     struct amdgpu_device *adev = rd->adev;
+     int result = 0, r;
+     uint32_t value;
+
+     if (rd->state.offset & 0x3)
+             return -EINVAL;
+
+     if (rd->state.use_grbm) {
+             if (rd->state.grbm.se == 0x3FF)
+                     rd->state.grbm.se = 0xFFFFFFFF;
+             if (rd->state.grbm.sh == 0x3FF)
+                     rd->state.grbm.sh = 0xFFFFFFFF;
+             if (rd->state.grbm.instance == 0x3FF)
+                     rd->state.grbm.instance = 0xFFFFFFFF;
+     }
+
+     r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +10 -0
+     }
+
+     r = amdgpu_virt_enable_access_debugfs(adev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     if (rd->state.use_grbm) {
+             if ((rd->state.grbm.sh != 0xFFFFFFFF && rd->state.grbm.sh

= adev->gfx.config.max_sh_per_se) ||

Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 334

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                 (rd->state.grbm.se != 0xFFFFFFFF && rd->state.grbm.se

= adev->gfx.config.max_shader_engines)) {

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +2 -0
+                     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+

pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);

Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +6 -0
+                     amdgpu_virt_disable_access_debugfs(adev);
+                     return -EINVAL;
+             }
+             mutex_lock(&adev->grbm_idx_mutex);
+             amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se,
+                                                             rd->

state.grbm.sh,

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 343

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+

rd->state.grbm.instance);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +3 -0
+     } else if (rd->state.use_grbm) {
+             mutex_lock(&adev->srbm_mutex);
+             amdgpu_gfx_select_me_pipe_q(adev, rd->state.srbm.me,

rd->state.srbm.pipe,

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 347

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+

rd->state.srbm.queue, rd->state.srbm.vmid);

Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +4 -0
+     }
+
+     if (rd->state.pg_lock)
+             mutex_lock(&adev->pm.mutex);
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+
+     if (!write_en) {
+             value = RREG32(rd->state.offset >> 2);
+             r = put_user(value, (uint32_t *)buf);
+     } else {
+             r = get_user(value, (uint32_t *)buf);
+             if (!r)
+                     amdgpu_mm_wreg_mmio_rlc(adev, rd->state.offset >>

2, value);

Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -0
+     }
+     if (r) {
+             result = r;
+             goto end;
+     }
+     result = 0;
+end:
+     if (rd->state.use_grbm) {
+             amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff,

0xffffffff);

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +10 -0
+             mutex_unlock(&adev->grbm_idx_mutex);
+     } else if (rd->state.use_srbm) {
+             amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
+             mutex_unlock(&adev->srbm_mutex);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_unlock(&adev->pm.mutex);
+
+     // in umr (the likely user of this) flags are set per file

operation

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 379

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+     // which means they're never "unset" explicitly.  To avoid

breaking

Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 381

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +2 -0
+     // this convention we unset the flags after each operation
+     // flags are for a single call (need to be set for every

read/write)

Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +4 -0
+     rd->state.use_grbm = 0;
+     rd->state.use_srbm = 0;
+     rd->state.pg_lock  = 0;
+
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+     amdgpu_virt_disable_access_debugfs(adev);
+     return result;
+}
+
+static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int

cmd, unsigned long data)

Quoted 9 lines
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -0
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     unsigned char st[32], v;
+     int r, x;
+
+     // always read first 4 bytes of data
+     for (x = 0; x < 4; x++) {
+             if ((r = get_user(v, (unsigned char *)data))) {
+                     return r;
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+             }
+             ++data;
+             st[x] = v;
+     }
+
+     // first 4 bytes are offset
+     rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
+                                        ((u32)st[2] << 16) |

((u32)st[3] << 24);

Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +5 -0
+
+     switch (cmd) {
+             case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
+                     for (x = 4; x < 32; x++) {
+                             if ((r = get_user(v, (unsigned char

*)data))) {

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+                                     return r;
+                             }
+                             ++data;
+                             st[x] = v;
+                     }
+
+                     // next byte contains the flag
+                     // we only consume the remainder of the state if

bit 1 is set

Quoted 7 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +7 -0
+                     // this allows the subsequent read/write
+                     rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
+                     rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
+                     rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
+
+                     // next 6 bytes are grbm data
+                     rd->state.grbm.se       =  ((u32)st[5]) |

((u32)st[6] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 431

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.grbm.sh       =  ((u32)st[7]) |

((u32)st[8] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 432

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.grbm.instance =  ((u32)st[9]) |

((u32)st[10] << 8);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +3 -0
+
+                     // next 8 are srbm data
+                     rd->state.srbm.me       =  ((u32)st[11]) |

((u32)st[12] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 436

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.srbm.pipe     =  ((u32)st[13]) |

((u32)st[14] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 437

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.srbm.queue    =  ((u32)st[15]) |

((u32)st[16] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 438

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.srbm.vmid     =  ((u32)st[17]) |

((u32)st[18] << 8);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +3 -0
+                     break;
+             case AMDGPU_DEBUGFS_REGS2_IOC_READ:
+                     return amdgpu_debugfs_regs2_op(f, (char __user

*)data, 0);

Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 443

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +2 -0
+             case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
+                     return amdgpu_debugfs_regs2_op(f, (char __user

*)data, 1);

Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_gfxoff_read(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +5 -0
+             default:
+                     return -EINVAL;
+     }
+     return 0;
+}
 
   /**
    * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
@@ -1091,6 +1258,14 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct

file *f, char __user *buf,

Quoted 15 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static const struct file_operations

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
       return result;
   }
 
+static const struct file_operations amdgpu_debugfs_regs2_fops = {
+     .owner = THIS_MODULE,
+     .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
+     .open = amdgpu_debugfs_regs2_open,
+     .release = amdgpu_debugfs_regs2_release,
+     .llseek = default_llseek
+};
+
   static const struct file_operations amdgpu_debugfs_regs_fops = {
       .owner = THIS_MODULE,
       .read = amdgpu_debugfs_regs_read,
@@ -1148,6 +1323,7 @@ static const struct file_operations

amdgpu_debugfs_gfxoff_fops = {

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static const struct file_operations

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
   static const struct file_operations *debugfs_regs[] = {
       &amdgpu_debugfs_regs_fops,
+     &amdgpu_debugfs_regs2_fops,
       &amdgpu_debugfs_regs_didt_fops,
       &amdgpu_debugfs_regs_pcie_fops,
       &amdgpu_debugfs_regs_smc_fops,
@@ -1160,6 +1336,7 @@ static const struct file_operations

*debugfs_regs[] = {

Quoted 8 lines

Review

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
   static const char *debugfs_regs_names[] = {
       "amdgpu_regs",
+     "amdgpu_regs2",
       "amdgpu_regs_didt",
       "amdgpu_regs_pcie",
       "amdgpu_regs_smc",
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h

Quoted 9 lines
Quoted 9 lines

index 141a8474e24f..04c81cd4bcc7 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +1 -0
@@ -22,6 +22,7 @@
    * OTHER DEALINGS IN THE SOFTWARE.
    *
    */
+#include <linux/ioctl.h>
 
Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +0 -0
   /*
    * Debugfs
@@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device

*adev);

Quoted 31 lines
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +7 -0
   void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
   void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
   int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+
+struct amdgpu_debugfs_regs2_data {
+     struct amdgpu_device *adev;
+     struct {
+             // regs state
+             int use_srbm,
+                 use_grbm,

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 58

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +10 -0
+                 pg_lock;
+             struct {
+                     u32 se, sh, instance;
+             } grbm;
+             struct {
+                     u32 me, pipe, queue, vmid;
+             } srbm;
+             u32 offset;
+     } state;
+};

This stuff needs to be in some shared header then.

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 63

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +5 -0
+
+enum AMDGPU_DEBUGFS_REGS2_CMDS {
+     AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
+     AMDGPU_DEBUGFS_REGS2_CMD_READ,
+     AMDGPU_DEBUGFS_REGS2_CMD_WRITE,

Why not just using the normal read and write functions?

Christian.

Quoted 7 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +7 -0
+};
+
+struct amdgpu_debugfs_regs2_iocdata {
+     unsigned char t[32];
+};
+
+#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20,

AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct amdgpu_debugfs_regs2_iocdata)

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device · line 71

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +1 -0
+#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20,

AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device · line 72

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +1 -0
+#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20,

AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)

#5 lore raw

Am 24.08.21 um 14:42 schrieb Tom St Denis:

Quoted 1 line

The IOCTL data is in the debugfs header as it is. I could move that to the amdgpu_drm.h and include it from amdgpu_debugfs.h.

Na, keep it like that and just add a comment.

On second thought I don't want to raise any discussion on the mailing list if we should have a debugfs structure in the UAPI headers.

Christian.

Quoted 378 lines

I'll re-write the STATE IOCTL to use a struct and then test against what I have in umr.

Refactoring the read/write is trivial and I'll do that no problem (with style fixes).

Cheers,
Tom

On Tue, Aug 24, 2021 at 8:34 AM Christian König <[email protected] <mailto:[email protected]>> wrote:

    Am 24.08.21 um 14:27 schrieb StDenis, Tom:
    > [AMD Official Use Only]
    >
    > What do you mean a "shared header?"  How would they be shared
    between kernel and user?

    Somewhere in the include/uapi/drm/ folder I think. Either add that to
    amdgpu_drm.h or maybe amdgpu_debugfs.h?

    Or just keep it as a structure in amdgpu_debugfs.h with a comment
    that
    it is used for an IOCTL.

    Just not something hard coded like first byte is this, second that
    etc....

    > As for why not read/write.  Jus wanted to keep it simple.  It's
    not really performance bound.  umr never does reads/writes larger
    than 32-bits anyways.  It doesn't have to be this way though but I
    figured the fewer LOC the better.

    I think it would be cleaner if we would have separate functions
    for this.

    As far as I can see you also don't need the dance with the power
    managerment etc for the IOCTL. It's just get_user() into your
    structure.

    BTW: In the kernel coding style put "switch" and "case" on the same
    column, otherwise checkpatch.pl <http://checkpatch.pl> might complain.

    Christian.

    >
    > Tom
    >
    > ________________________________________
    > From: Christian König <[email protected]
    <mailto:[email protected]>>
    > Sent: Tuesday, August 24, 2021 08:20
    > To: StDenis, Tom; [email protected]
    <mailto:[email protected]>
    > Subject: Re: [PATCH] drm/amd/amdgpu: New debugfs interface for
    MMIO registers
    >
    >
    >
    > Am 24.08.21 um 14:16 schrieb Tom St Denis:
    >> This new debugfs interface uses an IOCTL interface in order to pass
    >> along state information like SRBM and GRBM bank switching.  This
    >> new interface also allows a full 32-bit MMIO address range which
    >> the previous didn't.  With this new design we have room to grow
    >> the flexibility of the file as need be.
    >>
    >> Signed-off-by: Tom St Denis <[email protected]
    <mailto:[email protected]>>
    >> ---
    >>    drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177
    ++++++++++++++++++++
    >>    drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
    >>    2 files changed, 209 insertions(+)
    >>
    >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
    b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
    >> index 277128846dd1..ab2d92f84da5 100644
    >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
    >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
    >> @@ -279,6 +279,173 @@ static ssize_t
    amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
    >>        return amdgpu_debugfs_process_reg_op(false, f, (char
    __user *)buf, size, pos);
    >>    }
    >>
    >> +static int amdgpu_debugfs_regs2_open(struct inode *inode,
    struct file *file)
    >> +{
    >> +     struct amdgpu_debugfs_regs2_data *rd;
    >> +
    >> +     rd = kzalloc(sizeof *rd, GFP_KERNEL);
    >> +     if (!rd)
    >> +             return -ENOMEM;
    >> +     rd->adev = file_inode(file)->i_private;
    >> +     file->private_data = rd;
    >> +
    >> +     return 0;
    >> +}
    >> +
    >> +static int amdgpu_debugfs_regs2_release(struct inode *inode,
    struct file *file)
    >> +{
    >> +     kfree(file->private_data);
    >> +     return 0;
    >> +}
    >> +
    >> +static int amdgpu_debugfs_regs2_op(struct file *f, char __user
    *buf, int write_en)
    >> +{
    >> +     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
    >> +     struct amdgpu_device *adev = rd->adev;
    >> +     int result = 0, r;
    >> +     uint32_t value;
    >> +
    >> +     if (rd->state.offset & 0x3)
    >> +             return -EINVAL;
    >> +
    >> +     if (rd->state.use_grbm) {
    >> +             if (rd->state.grbm.se <http://state.grbm.se> ==
    0x3FF)
    >> +                     rd->state.grbm.se <http://state.grbm.se>
    = 0xFFFFFFFF;
    >> +             if (rd->state.grbm.sh <http://state.grbm.sh> ==
    0x3FF)
    >> +                     rd->state.grbm.sh <http://state.grbm.sh>
    = 0xFFFFFFFF;
    >> +             if (rd->state.grbm.instance == 0x3FF)
    >> +                     rd->state.grbm.instance = 0xFFFFFFFF;
    >> +     }
    >> +
    >> +     r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
    >> +     if (r < 0) {
    >> +  pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
    >> +             return r;
    >> +     }
    >> +
    >> +     r = amdgpu_virt_enable_access_debugfs(adev);
    >> +     if (r < 0) {
    >> +  pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
    >> +             return r;
    >> +     }
    >> +
    >> +     if (rd->state.use_grbm) {
    >> +             if ((rd->state.grbm.sh <http://state.grbm.sh> !=
    0xFFFFFFFF && rd->state.grbm.sh <http://state.grbm.sh> >=
    adev->gfx.config.max_sh_per_se) ||
    >> +                 (rd->state.grbm.se <http://state.grbm.se> !=
    0xFFFFFFFF && rd->state.grbm.se <http://state.grbm.se> >=
    adev->gfx.config.max_shader_engines)) {
    >> +  pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
    >> +  pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
    >> +  amdgpu_virt_disable_access_debugfs(adev);
    >> +                     return -EINVAL;
    >> +             }
    >> +  mutex_lock(&adev->grbm_idx_mutex);
    >> +             amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se
    <http://state.grbm.se>,
    >> +          rd->state.grbm.sh <http://state.grbm.sh>,
    >> +          rd->state.grbm.instance);
    >> +     } else if (rd->state.use_grbm) {
    >> +             mutex_lock(&adev->srbm_mutex);
    >> +             amdgpu_gfx_select_me_pipe_q(adev,
    rd->state.srbm.me <http://state.srbm.me>, rd->state.srbm.pipe,
    >> +                  rd->state.srbm.queue, rd->state.srbm.vmid);
    >> +     }
    >> +
    >> +     if (rd->state.pg_lock)
    >> +             mutex_lock(&adev->pm.mutex);
    >> +
    >> +     if (!write_en) {
    >> +             value = RREG32(rd->state.offset >> 2);
    >> +             r = put_user(value, (uint32_t *)buf);
    >> +     } else {
    >> +             r = get_user(value, (uint32_t *)buf);
    >> +             if (!r)
    >> +                     amdgpu_mm_wreg_mmio_rlc(adev,
    rd->state.offset >> 2, value);
    >> +     }
    >> +     if (r) {
    >> +             result = r;
    >> +             goto end;
    >> +     }
    >> +     result = 0;
    >> +end:
    >> +     if (rd->state.use_grbm) {
    >> +             amdgpu_gfx_select_se_sh(adev, 0xffffffff,
    0xffffffff, 0xffffffff);
    >> +  mutex_unlock(&adev->grbm_idx_mutex);
    >> +     } else if (rd->state.use_srbm) {
    >> +             amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
    >> +             mutex_unlock(&adev->srbm_mutex);
    >> +     }
    >> +
    >> +     if (rd->state.pg_lock)
    >> +             mutex_unlock(&adev->pm.mutex);
    >> +
    >> +     // in umr (the likely user of this) flags are set per
    file operation
    >> +     // which means they're never "unset" explicitly.  To
    avoid breaking
    >> +     // this convention we unset the flags after each operation
    >> +     // flags are for a single call (need to be set for every
    read/write)
    >> +     rd->state.use_grbm = 0;
    >> +     rd->state.use_srbm = 0;
    >> +     rd->state.pg_lock  = 0;
    >> +
    >> +  pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
    >> +  pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
    >> +
    >> +     amdgpu_virt_disable_access_debugfs(adev);
    >> +     return result;
    >> +}
    >> +
    >> +static long amdgpu_debugfs_regs2_ioctl(struct file *f,
    unsigned int cmd, unsigned long data)
    >> +{
    >> +     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
    >> +     unsigned char st[32], v;
    >> +     int r, x;
    >> +
    >> +     // always read first 4 bytes of data
    >> +     for (x = 0; x < 4; x++) {
    >> +             if ((r = get_user(v, (unsigned char *)data))) {
    >> +                     return r;
    >> +             }
    >> +             ++data;
    >> +             st[x] = v;
    >> +     }
    >> +
    >> +     // first 4 bytes are offset
    >> +     rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
    >> +                                        ((u32)st[2] << 16) |
    ((u32)st[3] << 24);
    >> +
    >> +     switch (cmd) {
    >> +             case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
    >> +                     for (x = 4; x < 32; x++) {
    >> +                             if ((r = get_user(v, (unsigned
    char *)data))) {
    >> +                                     return r;
    >> +                             }
    >> +                             ++data;
    >> +                             st[x] = v;
    >> +                     }
    >> +
    >> +                     // next byte contains the flag
    >> +                     // we only consume the remainder of the
    state if bit 1 is set
    >> +                     // this allows the subsequent read/write
    >> +                     rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
    >> +                     rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
    >> +                     rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
    >> +
    >> +                     // next 6 bytes are grbm data
    >> +                     rd->state.grbm.se <http://state.grbm.se> 
         = ((u32)st[5]) | ((u32)st[6] << 8);
    >> +                     rd->state.grbm.sh <http://state.grbm.sh> 
         = ((u32)st[7]) | ((u32)st[8] << 8);
    >> +                     rd->state.grbm.instance = ((u32)st[9]) |
    ((u32)st[10] << 8);
    >> +
    >> +                     // next 8 are srbm data
    >> +                     rd->state.srbm.me <http://state.srbm.me> 
         = ((u32)st[11]) | ((u32)st[12] << 8);
    >> +                     rd->state.srbm.pipe     = ((u32)st[13]) |
    ((u32)st[14] << 8);
    >> +                     rd->state.srbm.queue    = ((u32)st[15]) |
    ((u32)st[16] << 8);
    >> +                     rd->state.srbm.vmid     = ((u32)st[17]) |
    ((u32)st[18] << 8);
    >> +                     break;
    >> +             case AMDGPU_DEBUGFS_REGS2_IOC_READ:
    >> +                     return amdgpu_debugfs_regs2_op(f, (char
    __user *)data, 0);
    >> +             case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
    >> +                     return amdgpu_debugfs_regs2_op(f, (char
    __user *)data, 1);
    >> +             default:
    >> +                     return -EINVAL;
    >> +     }
    >> +     return 0;
    >> +}
    >>
    >>    /**
    >>     * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
    >> @@ -1091,6 +1258,14 @@ static ssize_t
    amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
    >>        return result;
    >>    }
    >>
    >> +static const struct file_operations amdgpu_debugfs_regs2_fops = {
    >> +     .owner = THIS_MODULE,
    >> +     .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
    >> +     .open = amdgpu_debugfs_regs2_open,
    >> +     .release = amdgpu_debugfs_regs2_release,
    >> +     .llseek = default_llseek
    >> +};
    >> +
    >>    static const struct file_operations amdgpu_debugfs_regs_fops = {
    >>        .owner = THIS_MODULE,
    >>        .read = amdgpu_debugfs_regs_read,
    >> @@ -1148,6 +1323,7 @@ static const struct file_operations
    amdgpu_debugfs_gfxoff_fops = {
    >>
    >>    static const struct file_operations *debugfs_regs[] = {
    >>        &amdgpu_debugfs_regs_fops,
    >> +     &amdgpu_debugfs_regs2_fops,
    >>        &amdgpu_debugfs_regs_didt_fops,
    >>        &amdgpu_debugfs_regs_pcie_fops,
    >>        &amdgpu_debugfs_regs_smc_fops,
    >> @@ -1160,6 +1336,7 @@ static const struct file_operations
    *debugfs_regs[] = {
    >>
    >>    static const char *debugfs_regs_names[] = {
    >>        "amdgpu_regs",
    >> +     "amdgpu_regs2",
    >>        "amdgpu_regs_didt",
    >>        "amdgpu_regs_pcie",
    >>        "amdgpu_regs_smc",
    >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
    b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
    >> index 141a8474e24f..04c81cd4bcc7 100644
    >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
    >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
    >> @@ -22,6 +22,7 @@
    >>     * OTHER DEALINGS IN THE SOFTWARE.
    >>     *
    >>     */
    >> +#include <linux/ioctl.h>
    >>
    >>    /*
    >>     * Debugfs
    >> @@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct
    amdgpu_device *adev);
    >>    void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
    >>    void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
    >>    int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
    >> +
    >> +struct amdgpu_debugfs_regs2_data {
    >> +     struct amdgpu_device *adev;
    >> +     struct {
    >> +             // regs state
    >> +             int use_srbm,
    >> +                 use_grbm,
    >> +                 pg_lock;
    >> +             struct {
    >> +                     u32 se, sh, instance;
    >> +             } grbm;
    >> +             struct {
    >> +                     u32 me, pipe, queue, vmid;
    >> +             } srbm;
    >> +             u32 offset;
    >> +     } state;
    >> +};
    > This stuff needs to be in some shared header then.
    >
    >> +
    >> +enum AMDGPU_DEBUGFS_REGS2_CMDS {
    >> +     AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
    >> +     AMDGPU_DEBUGFS_REGS2_CMD_READ,
    >> +     AMDGPU_DEBUGFS_REGS2_CMD_WRITE,
    > Why not just using the normal read and write functions?
    >
    > Christian.
    >
    >> +};
    >> +
    >> +struct amdgpu_debugfs_regs2_iocdata {
    >> +     unsigned char t[32];
    >> +};
    >> +
    >> +#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20,
    AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct
    amdgpu_debugfs_regs2_iocdata)
    >> +#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20,
    AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)
    >> +#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20,
    AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)
#6 lore raw

hehehe I just moved it to uapi... No worries, you're the maintainer, I'll move it back before posting v2.

Cheers,
Tom

Quoted 386 lines · Christian König

Am 24.08.21 um 14:42 schrieb Tom St Denis:

The IOCTL data is in the debugfs header as it is. I could move that to the amdgpu_drm.h and include it from amdgpu_debugfs.h.

Na, keep it like that and just add a comment.

On second thought I don't want to raise any discussion on the mailing list if we should have a debugfs structure in the UAPI headers.

Christian.

I'll re-write the STATE IOCTL to use a struct and then test against what I have in umr.

Refactoring the read/write is trivial and I'll do that no problem (with style fixes).

Cheers,
Tom

Quoted 359 lines · Christian König

Am 24.08.21 um 14:27 schrieb StDenis, Tom:

Quoted 3 lines

[AMD Official Use Only]

What do you mean a "shared header?" How would they be shared between

kernel and user?

Somewhere in the include/uapi/drm/ folder I think. Either add that to amdgpu_drm.h or maybe amdgpu_debugfs.h?

Or just keep it as a structure in amdgpu_debugfs.h with a comment that it is used for an IOCTL.

Just not something hard coded like first byte is this, second that etc....

Quoted 1 line

As for why not read/write. Jus wanted to keep it simple. It's not

really performance bound. umr never does reads/writes larger than 32-bits anyways. It doesn't have to be this way though but I figured the fewer LOC the better.

I think it would be cleaner if we would have separate functions for this.

As far as I can see you also don't need the dance with the power managerment etc for the IOCTL. It's just get_user() into your structure.

BTW: In the kernel coding style put "switch" and "case" on the same column, otherwise checkpatch.pl might complain.

Christian.

Quoted 8 lines

Tom

________________________________________
From: Christian König <[email protected]>
Sent: Tuesday, August 24, 2021 08:20
To: StDenis, Tom; [email protected]
Subject: Re: [PATCH] drm/amd/amdgpu: New debugfs interface for MMIO

registers

Quoted 13 lines

Am 24.08.21 um 14:16 schrieb Tom St Denis:

Quoted 9 lines

This new debugfs interface uses an IOCTL interface in order to pass along state information like SRBM and GRBM bank switching. This new interface also allows a full 32-bit MMIO address range which the previous didn't. With this new design we have room to grow the flexibility of the file as need be.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177

++++++++++++++++++++

Quoted 4 lines

Review

Quoted 4 lines
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
   2 files changed, 209 insertions(+)
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct

Quoted 4 lines

index 277128846dd1..ab2d92f84da5 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +0 -0
@@ -279,6 +279,173 @@ static ssize_t amdgpu_debugfs_regs_write(struct

file *f, const char __user *buf,

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 279

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +0 -0
       return amdgpu_debugfs_process_reg_op(false, f, (char __user

*)buf, size, pos);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 282

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
   }
 
+static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file

*file)

Quoted 5 lines
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +5 -0
+{
+     struct amdgpu_debugfs_regs2_data *rd;
+
+     rd = kzalloc(sizeof *rd, GFP_KERNEL);
+     if (!rd)
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+             return -ENOMEM;
+     rd->adev = file_inode(file)->i_private;
+     file->private_data = rd;
+
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_release(struct inode *inode, struct

file *file)

Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +6 -0
+{
+     kfree(file->private_data);
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_op(struct file *f, char __user *buf,

int write_en)

Quoted 24 lines
Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +24 -0
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     struct amdgpu_device *adev = rd->adev;
+     int result = 0, r;
+     uint32_t value;
+
+     if (rd->state.offset & 0x3)
+             return -EINVAL;
+
+     if (rd->state.use_grbm) {
+             if (rd->state.grbm.se == 0x3FF)
+                     rd->state.grbm.se = 0xFFFFFFFF;
+             if (rd->state.grbm.sh == 0x3FF)
+                     rd->state.grbm.sh = 0xFFFFFFFF;
+             if (rd->state.grbm.instance == 0x3FF)
+                     rd->state.grbm.instance = 0xFFFFFFFF;
+     }
+
+     r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+     r = amdgpu_virt_enable_access_debugfs(adev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     if (rd->state.use_grbm) {
+             if ((rd->state.grbm.sh != 0xFFFFFFFF && rd->

state.grbm.sh >= adev->gfx.config.max_sh_per_se) ||

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 334

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                 (rd->state.grbm.se != 0xFFFFFFFF && rd->

state.grbm.se >= adev->gfx.config.max_shader_engines)) {

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 335

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+

pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 336

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+

pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);

Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +6 -0
+                     amdgpu_virt_disable_access_debugfs(adev);
+                     return -EINVAL;
+             }
+             mutex_lock(&adev->grbm_idx_mutex);
+             amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se,
+                                                             rd->

state.grbm.sh,

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 343

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+

rd->state.grbm.instance);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +3 -0
+     } else if (rd->state.use_grbm) {
+             mutex_lock(&adev->srbm_mutex);
+             amdgpu_gfx_select_me_pipe_q(adev, rd->state.srbm.me,

rd->state.srbm.pipe,

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 347

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+

rd->state.srbm.queue, rd->state.srbm.vmid);

Quoted 14 lines
Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +13 -0
+     }
+
+     if (rd->state.pg_lock)
+             mutex_lock(&adev->pm.mutex);
+
+     if (!write_en) {
+             value = RREG32(rd->state.offset >> 2);
+             r = put_user(value, (uint32_t *)buf);
+     } else {
+             r = get_user(value, (uint32_t *)buf);
+             if (!r)
+                     amdgpu_mm_wreg_mmio_rlc(adev, rd->state.offset
 2, value);
+     }
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+     if (r) {
+             result = r;
+             goto end;
+     }
+     result = 0;
+end:
+     if (rd->state.use_grbm) {
+             amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff,

0xffffffff);

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +10 -0
+             mutex_unlock(&adev->grbm_idx_mutex);
+     } else if (rd->state.use_srbm) {
+             amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
+             mutex_unlock(&adev->srbm_mutex);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_unlock(&adev->pm.mutex);
+
+     // in umr (the likely user of this) flags are set per file

operation

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 380

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+     // which means they're never "unset" explicitly.  To avoid

breaking

Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 382

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +2 -0
+     // this convention we unset the flags after each operation
+     // flags are for a single call (need to be set for every

read/write)

Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +4 -0
+     rd->state.use_grbm = 0;
+     rd->state.use_srbm = 0;
+     rd->state.pg_lock  = 0;
+
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+     amdgpu_virt_disable_access_debugfs(adev);
+     return result;
+}
+
+static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int

cmd, unsigned long data)

Quoted 9 lines
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -0
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     unsigned char st[32], v;
+     int r, x;
+
+     // always read first 4 bytes of data
+     for (x = 0; x < 4; x++) {
+             if ((r = get_user(v, (unsigned char *)data))) {
+                     return r;
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+             }
+             ++data;
+             st[x] = v;
+     }
+
+     // first 4 bytes are offset
+     rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
+                                        ((u32)st[2] << 16) |

((u32)st[3] << 24);

Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +5 -0
+
+     switch (cmd) {
+             case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
+                     for (x = 4; x < 32; x++) {
+                             if ((r = get_user(v, (unsigned char

*)data))) {

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
+                                     return r;
+                             }
+                             ++data;
+                             st[x] = v;
+                     }
+
+                     // next byte contains the flag
+                     // we only consume the remainder of the state if

bit 1 is set

Quoted 7 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +7 -0
+                     // this allows the subsequent read/write
+                     rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
+                     rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
+                     rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
+
+                     // next 6 bytes are grbm data
+                     rd->state.grbm.se       =  ((u32)st[5]) |

((u32)st[6] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 432

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.grbm.sh       =  ((u32)st[7]) |

((u32)st[8] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 433

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.grbm.instance =  ((u32)st[9]) |

((u32)st[10] << 8);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +3 -0
+
+                     // next 8 are srbm data
+                     rd->state.srbm.me       =  ((u32)st[11]) |

((u32)st[12] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 437

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.srbm.pipe     =  ((u32)st[13]) |

((u32)st[14] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 438

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.srbm.queue    =  ((u32)st[15]) |

((u32)st[16] << 8);

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 439

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
+                     rd->state.srbm.vmid     =  ((u32)st[17]) |

((u32)st[18] << 8);

Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +3 -0
+                     break;
+             case AMDGPU_DEBUGFS_REGS2_IOC_READ:
+                     return amdgpu_debugfs_regs2_op(f, (char __user

*)data, 0);

Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t amdgpu_debugfs_regs_write(struct · line 444

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +2 -0
+             case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
+                     return amdgpu_debugfs_regs2_op(f, (char __user

*)data, 1);

Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static ssize_t

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +5 -0
+             default:
+                     return -EINVAL;
+     }
+     return 0;
+}
 
   /**

* amdgpu_debugfs_regs_pcie_read - Read from a PCIE register

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +0 -0
@@ -1091,6 +1258,14 @@ static ssize_t

amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,

Quoted 15 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static const struct file_operations

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +8 -0
       return result;
   }
 
+static const struct file_operations amdgpu_debugfs_regs2_fops = {
+     .owner = THIS_MODULE,
+     .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
+     .open = amdgpu_debugfs_regs2_open,
+     .release = amdgpu_debugfs_regs2_release,
+     .llseek = default_llseek
+};
+
   static const struct file_operations amdgpu_debugfs_regs_fops = {
       .owner = THIS_MODULE,
       .read = amdgpu_debugfs_regs_read,
@@ -1148,6 +1323,7 @@ static const struct file_operations

amdgpu_debugfs_gfxoff_fops = {

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c static const struct file_operations

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
   static const struct file_operations *debugfs_regs[] = {
       &amdgpu_debugfs_regs_fops,
+     &amdgpu_debugfs_regs2_fops,
       &amdgpu_debugfs_regs_didt_fops,
       &amdgpu_debugfs_regs_pcie_fops,
       &amdgpu_debugfs_regs_smc_fops,
@@ -1160,6 +1336,7 @@ static const struct file_operations

*debugfs_regs[] = {

Quoted 8 lines

Review

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +1 -0
   static const char *debugfs_regs_names[] = {
       "amdgpu_regs",
+     "amdgpu_regs2",
       "amdgpu_regs_didt",
       "amdgpu_regs_pcie",
       "amdgpu_regs_smc",
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h

Quoted 9 lines
Quoted 9 lines

index 141a8474e24f..04c81cd4bcc7 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +1 -0
@@ -22,6 +22,7 @@
    * OTHER DEALINGS IN THE SOFTWARE.
    *
    */
+#include <linux/ioctl.h>
 
Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +0 -0
   /*
    * Debugfs
@@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device

*adev);

Quoted 31 lines
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +7 -0
   void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
   void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
   int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+
+struct amdgpu_debugfs_regs2_data {
+     struct amdgpu_device *adev;
+     struct {
+             // regs state
+             int use_srbm,
+                 use_grbm,

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 58

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +10 -0
+                 pg_lock;
+             struct {
+                     u32 se, sh, instance;
+             } grbm;
+             struct {
+                     u32 me, pipe, queue, vmid;
+             } srbm;
+             u32 offset;
+     } state;
+};

This stuff needs to be in some shared header then.

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 63

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +5 -0
+
+enum AMDGPU_DEBUGFS_REGS2_CMDS {
+     AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
+     AMDGPU_DEBUGFS_REGS2_CMD_READ,
+     AMDGPU_DEBUGFS_REGS2_CMD_WRITE,

Why not just using the normal read and write functions?

Christian.

Quoted 7 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +7 -0
+};
+
+struct amdgpu_debugfs_regs2_iocdata {
+     unsigned char t[32];
+};
+
+#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20,

AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct amdgpu_debugfs_regs2_iocdata)

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device · line 71

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +1 -0
+#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20,

AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)

Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device · line 72

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +1 -0
+#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20,

AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)

#7 lore raw

Also, please use C style comments.

Alex

Quoted 342 lines · Tom St Denis

hehehe I just moved it to uapi... No worries, you're the maintainer, I'll move it back before posting v2.

Cheers,
Tom

Quoted 335 lines · Christian König

Am 24.08.21 um 14:42 schrieb Tom St Denis:

The IOCTL data is in the debugfs header as it is. I could move that to the amdgpu_drm.h and include it from amdgpu_debugfs.h.

Na, keep it like that and just add a comment.

On second thought I don't want to raise any discussion on the mailing list if we should have a debugfs structure in the UAPI headers.

Christian.

I'll re-write the STATE IOCTL to use a struct and then test against what I have in umr.

Refactoring the read/write is trivial and I'll do that no problem (with style fixes).

Cheers,
Tom

Quoted 313 lines · Christian König

Am 24.08.21 um 14:27 schrieb StDenis, Tom:

Quoted 3 lines

[AMD Official Use Only]

What do you mean a "shared header?" How would they be shared between kernel and user?

Somewhere in the include/uapi/drm/ folder I think. Either add that to amdgpu_drm.h or maybe amdgpu_debugfs.h?

Or just keep it as a structure in amdgpu_debugfs.h with a comment that it is used for an IOCTL.

Just not something hard coded like first byte is this, second that etc....

Quoted 1 line

As for why not read/write. Jus wanted to keep it simple. It's not really performance bound. umr never does reads/writes larger than 32-bits anyways. It doesn't have to be this way though but I figured the fewer LOC the better.

I think it would be cleaner if we would have separate functions for this.

As far as I can see you also don't need the dance with the power managerment etc for the IOCTL. It's just get_user() into your structure.

BTW: In the kernel coding style put "switch" and "case" on the same column, otherwise checkpatch.pl might complain.

Christian.

Quoted 286 lines

Tom

________________________________________
From: Christian König <[email protected]>
Sent: Tuesday, August 24, 2021 08:20
To: StDenis, Tom; [email protected]
Subject: Re: [PATCH] drm/amd/amdgpu: New debugfs interface for MMIO registers

Am 24.08.21 um 14:16 schrieb Tom St Denis:

Quoted 244 lines

This new debugfs interface uses an IOCTL interface in order to pass along state information like SRBM and GRBM bank switching. This new interface also allows a full 32-bit MMIO address range which the previous didn't. With this new design we have room to grow the flexibility of the file as need be.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 177 ++++++++++++++++++++
   drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h |  32 ++++
   2 files changed, 209 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +177 -0
index 277128846dd1..ab2d92f84da5 100644
@@ -279,6 +279,173 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
       return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
   }
 
+static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
+{
+     struct amdgpu_debugfs_regs2_data *rd;
+
+     rd = kzalloc(sizeof *rd, GFP_KERNEL);
+     if (!rd)
+             return -ENOMEM;
+     rd->adev = file_inode(file)->i_private;
+     file->private_data = rd;
+
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
+{
+     kfree(file->private_data);
+     return 0;
+}
+
+static int amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, int write_en)
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     struct amdgpu_device *adev = rd->adev;
+     int result = 0, r;
+     uint32_t value;
+
+     if (rd->state.offset & 0x3)
+             return -EINVAL;
+
+     if (rd->state.use_grbm) {
+             if (rd->state.grbm.se == 0x3FF)
+                     rd->state.grbm.se = 0xFFFFFFFF;
+             if (rd->state.grbm.sh == 0x3FF)
+                     rd->state.grbm.sh = 0xFFFFFFFF;
+             if (rd->state.grbm.instance == 0x3FF)
+                     rd->state.grbm.instance = 0xFFFFFFFF;
+     }
+
+     r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     r = amdgpu_virt_enable_access_debugfs(adev);
+     if (r < 0) {
+             pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+             return r;
+     }
+
+     if (rd->state.use_grbm) {
+             if ((rd->state.grbm.sh != 0xFFFFFFFF && rd->state.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
+                 (rd->state.grbm.se != 0xFFFFFFFF && rd->state.grbm.se >= adev->gfx.config.max_shader_engines)) {
+                     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+                     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+                     amdgpu_virt_disable_access_debugfs(adev);
+                     return -EINVAL;
+             }
+             mutex_lock(&adev->grbm_idx_mutex);
+             amdgpu_gfx_select_se_sh(adev, rd->state.grbm.se,
+                                                             rd->state.grbm.sh,
+                                                             rd->state.grbm.instance);
+     } else if (rd->state.use_grbm) {
+             mutex_lock(&adev->srbm_mutex);
+             amdgpu_gfx_select_me_pipe_q(adev, rd->state.srbm.me, rd->state.srbm.pipe,
+                                                                     rd->state.srbm.queue, rd->state.srbm.vmid);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_lock(&adev->pm.mutex);
+
+     if (!write_en) {
+             value = RREG32(rd->state.offset >> 2);
+             r = put_user(value, (uint32_t *)buf);
+     } else {
+             r = get_user(value, (uint32_t *)buf);
+             if (!r)
+                     amdgpu_mm_wreg_mmio_rlc(adev, rd->state.offset >> 2, value);
+     }
+     if (r) {
+             result = r;
+             goto end;
+     }
+     result = 0;
+end:
+     if (rd->state.use_grbm) {
+             amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+             mutex_unlock(&adev->grbm_idx_mutex);
+     } else if (rd->state.use_srbm) {
+             amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
+             mutex_unlock(&adev->srbm_mutex);
+     }
+
+     if (rd->state.pg_lock)
+             mutex_unlock(&adev->pm.mutex);
+
+     // in umr (the likely user of this) flags are set per file operation
+     // which means they're never "unset" explicitly.  To avoid breaking
+     // this convention we unset the flags after each operation
+     // flags are for a single call (need to be set for every read/write)
+     rd->state.use_grbm = 0;
+     rd->state.use_srbm = 0;
+     rd->state.pg_lock  = 0;
+
+     pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+     pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+     amdgpu_virt_disable_access_debugfs(adev);
+     return result;
+}
+
+static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
+{
+     struct amdgpu_debugfs_regs2_data *rd = f->private_data;
+     unsigned char st[32], v;
+     int r, x;
+
+     // always read first 4 bytes of data
+     for (x = 0; x < 4; x++) {
+             if ((r = get_user(v, (unsigned char *)data))) {
+                     return r;
+             }
+             ++data;
+             st[x] = v;
+     }
+
+     // first 4 bytes are offset
+     rd->state.offset = ((u32)st[0]) | ((u32)st[1] << 8) |
+                                        ((u32)st[2] << 16) | ((u32)st[3] << 24);
+
+     switch (cmd) {
+             case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
+                     for (x = 4; x < 32; x++) {
+                             if ((r = get_user(v, (unsigned char *)data))) {
+                                     return r;
+                             }
+                             ++data;
+                             st[x] = v;
+                     }
+
+                     // next byte contains the flag
+                     // we only consume the remainder of the state if bit 1 is set
+                     // this allows the subsequent read/write
+                     rd->state.use_grbm = (st[4] & 1) ? 1 : 0;
+                     rd->state.use_srbm = (st[4] & 2) ? 1 : 0;
+                     rd->state.pg_lock  = (st[4] & 4) ? 1 : 0;
+
+                     // next 6 bytes are grbm data
+                     rd->state.grbm.se       =  ((u32)st[5]) | ((u32)st[6] << 8);
+                     rd->state.grbm.sh       =  ((u32)st[7]) | ((u32)st[8] << 8);
+                     rd->state.grbm.instance =  ((u32)st[9]) | ((u32)st[10] << 8);
+
+                     // next 8 are srbm data
+                     rd->state.srbm.me       =  ((u32)st[11]) | ((u32)st[12] << 8);
+                     rd->state.srbm.pipe     =  ((u32)st[13]) | ((u32)st[14] << 8);
+                     rd->state.srbm.queue    =  ((u32)st[15]) | ((u32)st[16] << 8);
+                     rd->state.srbm.vmid     =  ((u32)st[17]) | ((u32)st[18] << 8);
+                     break;
+             case AMDGPU_DEBUGFS_REGS2_IOC_READ:
+                     return amdgpu_debugfs_regs2_op(f, (char __user *)data, 0);
+             case AMDGPU_DEBUGFS_REGS2_IOC_WRITE:
+                     return amdgpu_debugfs_regs2_op(f, (char __user *)data, 1);
+             default:
+                     return -EINVAL;
+     }
+     return 0;
+}
 
   /**
    * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
@@ -1091,6 +1258,14 @@ static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
       return result;
   }
 
+static const struct file_operations amdgpu_debugfs_regs2_fops = {
+     .owner = THIS_MODULE,
+     .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
+     .open = amdgpu_debugfs_regs2_open,
+     .release = amdgpu_debugfs_regs2_release,
+     .llseek = default_llseek
+};
+
   static const struct file_operations amdgpu_debugfs_regs_fops = {
       .owner = THIS_MODULE,
       .read = amdgpu_debugfs_regs_read,
@@ -1148,6 +1323,7 @@ static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
 
   static const struct file_operations *debugfs_regs[] = {
       &amdgpu_debugfs_regs_fops,
+     &amdgpu_debugfs_regs2_fops,
       &amdgpu_debugfs_regs_didt_fops,
       &amdgpu_debugfs_regs_pcie_fops,
       &amdgpu_debugfs_regs_smc_fops,
@@ -1160,6 +1336,7 @@ static const struct file_operations *debugfs_regs[] = {
 
   static const char *debugfs_regs_names[] = {
       "amdgpu_regs",
+     "amdgpu_regs2",
       "amdgpu_regs_didt",
       "amdgpu_regs_pcie",
       "amdgpu_regs_smc",
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +8 -0
index 141a8474e24f..04c81cd4bcc7 100644
@@ -22,6 +22,7 @@
    * OTHER DEALINGS IN THE SOFTWARE.
    *
    */
+#include <linux/ioctl.h>
 
   /*
    * Debugfs
@@ -38,3 +39,34 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
   void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
   void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
   int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+
+struct amdgpu_debugfs_regs2_data {
+     struct amdgpu_device *adev;
+     struct {
+             // regs state
+             int use_srbm,
+                 use_grbm,

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 58

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +10 -0
+                 pg_lock;
+             struct {
+                     u32 se, sh, instance;
+             } grbm;
+             struct {
+                     u32 me, pipe, queue, vmid;
+             } srbm;
+             u32 offset;
+     } state;
+};

This stuff needs to be in some shared header then.

drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h void amdgpu_debugfs_fence_init(struct amdgpu_device *adev); · line 63

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +5 -0
+
+enum AMDGPU_DEBUGFS_REGS2_CMDS {
+     AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE=0,
+     AMDGPU_DEBUGFS_REGS2_CMD_READ,
+     AMDGPU_DEBUGFS_REGS2_CMD_WRITE,

Why not just using the normal read and write functions?

Christian.

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h +9 -0
+};
+
+struct amdgpu_debugfs_regs2_iocdata {
+     unsigned char t[32];
+};
+
+#define AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_SET_STATE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_READ _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_WRITE, struct amdgpu_debugfs_regs2_iocdata)
+#define AMDGPU_DEBUGFS_REGS2_IOC_WRITE _IOWR(0x20, AMDGPU_DEBUGFS_REGS2_CMD_READ, struct amdgpu_debugfs_regs2_iocdata)