Skip to contentSkip to conversation

drm/amdgpu: Unmap MMIO mappings when device is not unplugged

PATCH In review Leslie Shi 2 participants 4 messages Jan 5, 2022 – Jan 6, 2022

Subsystems (2)
lore
Share & tools
lore

Leslie Shi

author
#0 lore raw

PATCH v2 1/1 drm/amdgpu: Unmap MMIO mappings when device is not unplugged

Patch: 3efb17ae7e92 ("drm/amdgpu: Call amdgpu_device_unmap_mmio() if device is unplugged to prevent crash in GPU initialization failure") makes call to amdgpu_device_unmap_mmio() conditioned on device unplugged. This patch unmaps MMIO mappings even when device is not unplugged.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 11 +++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 12 ++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    | 11 +++++++++++
 3 files changed, 34 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -0
index 412f377f80b1..16dc16c860cc 100644
@@ -3832,6 +3832,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 
 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
 {
+
 	/* Clear all CPU mappings pointing to this device */
 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
 
@@ -3912,6 +3913,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 
 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
 {
+	int idx;
+
 	amdgpu_fence_driver_sw_fini(adev);
 	amdgpu_device_ip_fini(adev);
 	release_firmware(adev->firmware.gpu_info_fw);
@@ -3936,6 +3939,14 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
 		vga_client_register(adev->pdev, NULL, NULL, NULL);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		iounmap(adev->rmmio);
+		adev->rmmio = NULL;
+		amdgpu_device_doorbell_fini(adev);
+		drm_dev_exit(idx);
+	}
+
 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
 		amdgpu_pmu_fini(adev);
 	if (adev->mman.discovery_bin)
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +12 -0
index 156002db24e1..ff9dc377a3a0 100644
@@ -33,6 +33,7 @@
 #include <linux/slab.h>
 #include <linux/dma-buf.h>
 
+#include <drm/drm_drv.h>
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_cache.h>
 #include "amdgpu.h"
@@ -1061,7 +1062,18 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
  */
 void amdgpu_bo_fini(struct amdgpu_device *adev)
 {
+	int idx;
+
 	amdgpu_ttm_fini(adev);
+
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (!adev->gmc.xgmi.connected_to_cpu) {
+			arch_phys_wc_del(adev->gmc.vram_mtrr);
+			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
+		}
+		drm_dev_exit(idx);
+	}
 }
 
 /**
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -0
index 367abed1d6e6..ea897feeddd2 100644
@@ -42,6 +42,7 @@
 #include <linux/dma-buf.h>
 #include <linux/sizes.h>
 
+#include <drm/drm_drv.h>
 #include <drm/ttm/ttm_bo_api.h>
 #include <drm/ttm/ttm_bo_driver.h>
 #include <drm/ttm/ttm_placement.h>
@@ -1801,6 +1802,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
  */
 void amdgpu_ttm_fini(struct amdgpu_device *adev)
 {
+	int idx;
 	if (!adev->mman.initialized)
 		return;
 
@@ -1815,6 +1817,15 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
 				      NULL, NULL);
 	amdgpu_ttm_fw_reserve_vram_fini(adev);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (adev->mman.aper_base_kaddr)
+			iounmap(adev->mman.aper_base_kaddr);
+		adev->mman.aper_base_kaddr = NULL;
+
+		drm_dev_exit(idx);
+	}
+
 	amdgpu_vram_mgr_fini(adev);
 	amdgpu_gtt_mgr_fini(adev);
 	amdgpu_preempt_mgr_fini(adev);
Signature · 1 line
2.25.1
#1 lore raw
Quoted 4 lines · Leslie Shi

Patch: 3efb17ae7e92 ("drm/amdgpu: Call amdgpu_device_unmap_mmio() if device is unplugged to prevent crash in GPU initialization failure") makes call to amdgpu_device_unmap_mmio() conditioned on device unplugged. This patch unmaps MMIO mappings even when device is not unplugged.

I don't see the 'call to amdgpu_device_unmap_mmio() conditioned on device unplugged' part in this patch

Also, please add 'v2:bla bla bla' part in patch description telling what was done in v2

Andrey

Quoted 111 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 11 +++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 12 ++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    | 11 +++++++++++
  3 files changed, 34 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -0
index 412f377f80b1..16dc16c860cc 100644
@@ -3832,6 +3832,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 
  static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
  {
+
  	/* Clear all CPU mappings pointing to this device */
  	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
 
@@ -3912,6 +3913,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 
  void amdgpu_device_fini_sw(struct amdgpu_device *adev)
  {
+	int idx;
+
  	amdgpu_fence_driver_sw_fini(adev);
  	amdgpu_device_ip_fini(adev);
  	release_firmware(adev->firmware.gpu_info_fw);
@@ -3936,6 +3939,14 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
  	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
  		vga_client_register(adev->pdev, NULL, NULL, NULL);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		iounmap(adev->rmmio);
+		adev->rmmio = NULL;
+		amdgpu_device_doorbell_fini(adev);
+		drm_dev_exit(idx);
+	}
+
  	if (IS_ENABLED(CONFIG_PERF_EVENTS))
  		amdgpu_pmu_fini(adev);
  	if (adev->mman.discovery_bin)
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +12 -0
index 156002db24e1..ff9dc377a3a0 100644
@@ -33,6 +33,7 @@
  #include <linux/slab.h>
  #include <linux/dma-buf.h>
 
+#include <drm/drm_drv.h>
  #include <drm/amdgpu_drm.h>
  #include <drm/drm_cache.h>
  #include "amdgpu.h"
@@ -1061,7 +1062,18 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
   */
  void amdgpu_bo_fini(struct amdgpu_device *adev)
  {
+	int idx;
+
  	amdgpu_ttm_fini(adev);
+
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (!adev->gmc.xgmi.connected_to_cpu) {
+			arch_phys_wc_del(adev->gmc.vram_mtrr);
+			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
+		}
+		drm_dev_exit(idx);
+	}
  }
 
  /**
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -0
index 367abed1d6e6..ea897feeddd2 100644
@@ -42,6 +42,7 @@
  #include <linux/dma-buf.h>
  #include <linux/sizes.h>
 
+#include <drm/drm_drv.h>
  #include <drm/ttm/ttm_bo_api.h>
  #include <drm/ttm/ttm_bo_driver.h>
  #include <drm/ttm/ttm_placement.h>
@@ -1801,6 +1802,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
   */
  void amdgpu_ttm_fini(struct amdgpu_device *adev)
  {
+	int idx;
  	if (!adev->mman.initialized)
  		return;
 
@@ -1815,6 +1817,15 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
  				      NULL, NULL);
  	amdgpu_ttm_fw_reserve_vram_fini(adev);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (adev->mman.aper_base_kaddr)
+			iounmap(adev->mman.aper_base_kaddr);
+		adev->mman.aper_base_kaddr = NULL;
+
+		drm_dev_exit(idx);
+	}
+
  	amdgpu_vram_mgr_fini(adev);
  	amdgpu_gtt_mgr_fini(adev);
  	amdgpu_preempt_mgr_fini(adev);
#2 lore raw

[AMD Official Use Only]

Hi Andrey,

It is the following patch calls to amdgpu_device_unmap_mmio() conditioned on device unplugged.

3efb17ae7e92 "drm/amdgpu: Call amdgpu_device_unmap_mmio() if device is unplugged to prevent crash in GPU initialization failure"

Regards,
Leslie

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Thursday, January 6, 2022 2:22 AM
To: Shi, Leslie <[email protected]>; Lazar, Lijo <[email protected]>; [email protected]
Cc: Chen, Guchun <[email protected]>
Subject: Re: [PATCH v2] drm/amdgpu: Unmap MMIO mappings when device is not unplugged

Quoted 5 lines · Leslie Shi

Patch: 3efb17ae7e92 ("drm/amdgpu: Call amdgpu_device_unmap_mmio() if device is unplugged to prevent crash in GPU initialization failure") makes call to
amdgpu_device_unmap_mmio() conditioned on device unplugged. This patch unmaps MMIO mappings even when device is not unplugged.

I don't see the 'call to amdgpu_device_unmap_mmio() conditioned on device unplugged' part in this patch

Also, please add 'v2:bla bla bla' part in patch description telling what was done in v2

Andrey

Quoted 116 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 11 +++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 12 ++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    | 11 +++++++++++
  3 files changed, 34 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
index 412f377f80b1..16dc16c860cc 100644
@@ -3832,6 +3832,7 @@ int amdgpu_device_init(struct amdgpu_device 
 *adev,
 
  static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
  {
+
  	/* Clear all CPU mappings pointing to this device */
  	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
@@ -3912,6 +3913,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device 
 *adev)
 
  void amdgpu_device_fini_sw(struct amdgpu_device *adev)
  {
+	int idx;
+
  	amdgpu_fence_driver_sw_fini(adev);
  	amdgpu_device_ip_fini(adev);
  	release_firmware(adev->firmware.gpu_info_fw);
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +8 -0
@@ -3936,6 +3939,14 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
  	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
  		vga_client_register(adev->pdev, NULL, NULL, NULL);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		iounmap(adev->rmmio);
+		adev->rmmio = NULL;
+		amdgpu_device_doorbell_fini(adev);
+		drm_dev_exit(idx);
+	}
+
  	if (IS_ENABLED(CONFIG_PERF_EVENTS))
  		amdgpu_pmu_fini(adev);
  	if (adev->mman.discovery_bin)
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +12 -0
index 156002db24e1..ff9dc377a3a0 100644
@@ -33,6 +33,7 @@
  #include <linux/slab.h>
  #include <linux/dma-buf.h>
 
+#include <drm/drm_drv.h>
  #include <drm/amdgpu_drm.h>
  #include <drm/drm_cache.h>
  #include "amdgpu.h"
@@ -1061,7 +1062,18 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
   */
  void amdgpu_bo_fini(struct amdgpu_device *adev)
  {
+	int idx;
+
  	amdgpu_ttm_fini(adev);
+
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (!adev->gmc.xgmi.connected_to_cpu) {
+			arch_phys_wc_del(adev->gmc.vram_mtrr);
+			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
+		}
+		drm_dev_exit(idx);
+	}
  }
 
  /**
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -0
index 367abed1d6e6..ea897feeddd2 100644
@@ -42,6 +42,7 @@
  #include <linux/dma-buf.h>
  #include <linux/sizes.h>
 
+#include <drm/drm_drv.h>
  #include <drm/ttm/ttm_bo_api.h>
  #include <drm/ttm/ttm_bo_driver.h>
  #include <drm/ttm/ttm_placement.h>
@@ -1801,6 +1802,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
   */
  void amdgpu_ttm_fini(struct amdgpu_device *adev)
  {
+	int idx;
  	if (!adev->mman.initialized)
  		return;
 
@@ -1815,6 +1817,15 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
  				      NULL, NULL);
  	amdgpu_ttm_fw_reserve_vram_fini(adev);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (adev->mman.aper_base_kaddr)
+			iounmap(adev->mman.aper_base_kaddr);
+		adev->mman.aper_base_kaddr = NULL;
+
+		drm_dev_exit(idx);
+	}
+
  	amdgpu_vram_mgr_fini(adev);
  	amdgpu_gtt_mgr_fini(adev);
  	amdgpu_preempt_mgr_fini(adev);
#3 lore raw

Got it

See bellow one small comment, with that the patch is Reviewed-by: Andrey Grodzovsky <[email protected]>

Quoted 49 lines · Leslie

[AMD Official Use Only]

Hi Andrey,

It is the following patch calls to amdgpu_device_unmap_mmio() conditioned on device unplugged.

3efb17ae7e92 "drm/amdgpu: Call amdgpu_device_unmap_mmio() if device is unplugged to prevent crash in GPU initialization failure"

Regards,
Leslie

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Thursday, January 6, 2022 2:22 AM
To: Shi, Leslie <[email protected]>; Lazar, Lijo <[email protected]>; [email protected]
Cc: Chen, Guchun <[email protected]>
Subject: Re: [PATCH v2] drm/amdgpu: Unmap MMIO mappings when device is not unplugged

Quoted 5 lines · Leslie Shi

Patch: 3efb17ae7e92 ("drm/amdgpu: Call amdgpu_device_unmap_mmio() if device is unplugged to prevent crash in GPU initialization failure") makes call to
amdgpu_device_unmap_mmio() conditioned on device unplugged. This patch unmaps MMIO mappings even when device is not unplugged.

drm/amdgpu: Call amdgpu_device_unmap_mmio() if device is unplugged to prevent crash in GPU initialization failure

I don't see the 'call to amdgpu_device_unmap_mmio() conditioned on device unplugged' part in this patch

Also, please add 'v2:bla bla bla' part in patch description telling what was done in v2

Andrey

Quoted 15 lines
---
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 11 +++++++++++
   drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 12 ++++++++++++
   drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    | 11 +++++++++++
   3 files changed, 34 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index 412f377f80b1..16dc16c860cc 100644
@@ -3832,6 +3832,7 @@ int amdgpu_device_init(struct amdgpu_device
 *adev,
 
Quoted 3 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_init(struct amdgpu_device *adev, · line 3835

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
   static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
   {
+

Drop the new line

Andrey

Quoted 97 lines
Quoted 97 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -0
   	/* Clear all CPU mappings pointing to this device */
   	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
@@ -3912,6 +3913,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device
 *adev)
 
   void amdgpu_device_fini_sw(struct amdgpu_device *adev)
   {
+	int idx;
+
   	amdgpu_fence_driver_sw_fini(adev);
   	amdgpu_device_ip_fini(adev);
   	release_firmware(adev->firmware.gpu_info_fw);
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +8 -0
@@ -3936,6 +3939,14 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
   	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
   		vga_client_register(adev->pdev, NULL, NULL, NULL);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		iounmap(adev->rmmio);
+		adev->rmmio = NULL;
+		amdgpu_device_doorbell_fini(adev);
+		drm_dev_exit(idx);
+	}
+
   	if (IS_ENABLED(CONFIG_PERF_EVENTS))
   		amdgpu_pmu_fini(adev);
   	if (adev->mman.discovery_bin)
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +12 -0
index 156002db24e1..ff9dc377a3a0 100644
@@ -33,6 +33,7 @@
   #include <linux/slab.h>
   #include <linux/dma-buf.h>
 
+#include <drm/drm_drv.h>
   #include <drm/amdgpu_drm.h>
   #include <drm/drm_cache.h>
   #include "amdgpu.h"
@@ -1061,7 +1062,18 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
    */
   void amdgpu_bo_fini(struct amdgpu_device *adev)
   {
+	int idx;
+
   	amdgpu_ttm_fini(adev);
+
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (!adev->gmc.xgmi.connected_to_cpu) {
+			arch_phys_wc_del(adev->gmc.vram_mtrr);
+			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
+		}
+		drm_dev_exit(idx);
+	}
   }
 
   /**
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -0
index 367abed1d6e6..ea897feeddd2 100644
@@ -42,6 +42,7 @@
   #include <linux/dma-buf.h>
   #include <linux/sizes.h>
 
+#include <drm/drm_drv.h>
   #include <drm/ttm/ttm_bo_api.h>
   #include <drm/ttm/ttm_bo_driver.h>
   #include <drm/ttm/ttm_placement.h>
@@ -1801,6 +1802,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
    */
   void amdgpu_ttm_fini(struct amdgpu_device *adev)
   {
+	int idx;
   	if (!adev->mman.initialized)
   		return;
 
@@ -1815,6 +1817,15 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
   				      NULL, NULL);
   	amdgpu_ttm_fw_reserve_vram_fini(adev);
 
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+
+		if (adev->mman.aper_base_kaddr)
+			iounmap(adev->mman.aper_base_kaddr);
+		adev->mman.aper_base_kaddr = NULL;
+
+		drm_dev_exit(idx);
+	}
+
   	amdgpu_vram_mgr_fini(adev);
   	amdgpu_gtt_mgr_fini(adev);
   	amdgpu_preempt_mgr_fini(adev);