drm/amdgpu: change reset lock from mutex to rw_semaphore
Subsystems (2)
PATCH v2 1/1 drm/amdgpu: change reset lock from mutex to rw_semaphore
clients don't need reset-lock for synchronization when no GPU recovery.
v2:
change to return the return value of down_read_killable.
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
@@ -954,7 +954,7 @@ struct amdgpu_device {atomic_t in_gpu_reset;enum pp_mp1_state mp1_state; -struct mutex lock_reset; +struct rw_semaphore reset_sem;struct amdgpu_doorbell_index doorbell_index;struct mutex notifier_lock;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +16 -7
@@ -101,14 +101,18 @@ static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file) file->private_data = adev; -mutex_lock (&adev->lock_reset); + ret =down_read_killable (&adev->reset_sem); +if (ret) +return ret; +if (adev->autodump.dumping.done) {reinit_completion (&adev->autodump.dumping); ret =0 ; }else { ret = -EBUSY ; } -mutex_unlock (&adev->lock_reset); + +up_read (&adev->reset_sem);return ret; }
@@ -1242,7 +1246,9 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) }/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock (&adev->lock_reset); + r =down_read_killable (&adev->reset_sem); +if (r) +return r;/* hold on the scheduler */ for (i =0 ; i <AMDGPU_MAX_RINGS ; i++) {
@@ -1269,7 +1275,7 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)kthread_unpark (ring->sched.thread); } -mutex_unlock (&adev->lock_reset); +up_read (&adev->reset_sem);pm_runtime_mark_last_busy (dev->dev);pm_runtime_put_autosuspend (dev->dev);
@@ -1459,7 +1465,9 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)return -ENOMEM ;/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock (&adev->lock_reset); + r =down_read_killable (&adev->reset_sem); +if (r) + goto pro_end;/* stop the scheduler */ kthread_park (ring->sched.thread);
@@ -1500,13 +1508,14 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)/* restart the scheduler */ kthread_unpark (ring->sched.thread); -mutex_unlock (&adev->lock_reset); +up_read (&adev->reset_sem);ttm_bo_unlock_delayed_workqueue (&adev->mman.bdev, resched); +pro_end:kfree (fences); -return 0 ; +return r; }static int amdgpu_debugfs_sclk_set (void *data,u64 val)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -3
@@ -3054,7 +3054,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,mutex_init (&adev->virt.vf_errors.lock);hash_init (adev->mn_hash);atomic_set (&adev->in_gpu_reset,0 ); -mutex_init (&adev->lock_reset); +init_rwsem (&adev->reset_sem);mutex_init (&adev->psp.mutex);mutex_init (&adev->notifier_lock);
@@ -4206,7 +4206,7 @@ static bool amdgpu_device_lock_adev(struct amdgpu_device *adev)if (atomic_cmpxchg (&adev->in_gpu_reset,0 ,1 ) !=0 )return false; -mutex_lock (&adev->lock_reset); +down_write (&adev->reset_sem);atomic_inc (&adev->gpu_reset_counter);switch (amdgpu_asic_reset_method (adev)) {
@@ -4229,7 +4229,7 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)amdgpu_vf_error_trans_all (adev); adev->mp1_state =PP_MP1_STATE_NONE ;atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); +up_write (&adev->reset_sem); }static void amdgpu_device_resume_display_audio (struct amdgpu_device *adev)
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -12
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);int timeout =AI_MAILBOX_POLL_FLR_TIMEDOUT ; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ - locked =mutex_trylock (&adev->lock_reset); -if (locked) -atomic_set (&adev->in_gpu_reset,1 ); +down_read (&adev->reset_sem); do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL)
@@ -261,10 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) }while (timeout >1 ); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); - } +up_read (&adev->reset_sem);/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +2 -12
@@ -259,19 +259,12 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);int timeout =NV_MAILBOX_POLL_FLR_TIMEDOUT ; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ - locked =mutex_trylock (&adev->lock_reset); -if (locked) -atomic_set (&adev->in_gpu_reset,1 ); +down_read (&adev->reset_sem); do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL)
@@ -282,10 +275,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) }while (timeout >1 ); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); - } +up_read (&adev->reset_sem);/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
Hi, Frank and Monk,
Can you help review this patch? Because it also changes some virtualization related codes.
Best Regards
Dennis Li
-----Original Message-----
From: Dennis Li <[email protected]>
Sent: Thursday, August 20, 2020 5:33 PM
To: [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Cc: Li, Dennis <[email protected]>
Subject: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
clients don't need reset-lock for synchronization when no GPU recovery.
v2:
change to return the return value of down_read_killable.
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
@@ -954,7 +954,7 @@ struct amdgpu_device {atomic_t in_gpu_reset;enum pp_mp1_state mp1_state; -struct mutex lock_reset; +struct rw_semaphore reset_sem;struct amdgpu_doorbell_index doorbell_index;struct mutex notifier_lock;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -3
@@ -101,14 +101,18 @@ static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file) file->private_data = adev; -mutex_lock (&adev->lock_reset); + ret =down_read_killable (&adev->reset_sem); +if (ret) +return ret; +if (adev->autodump.dumping.done) {reinit_completion (&adev->autodump.dumping); ret =0 ; }else { ret = -EBUSY ; } -mutex_unlock (&adev->lock_reset); + +up_read (&adev->reset_sem);return ret; }
@@ -1242,7 +1246,9 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) }/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock (&adev->lock_reset); + r =down_read_killable (&adev->reset_sem); +if (r) +return r;/* hold on the scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; i++) { @@ -1269,7 +1275,7 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
kthread_unpark(ring->sched.thread);
}
- mutex_unlock(&adev->lock_reset);
+ up_read(&adev->reset_sem);
pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev);
@@ -1459,7 +1465,9 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
return -ENOMEM;
/* Avoid accidently unparking the sched thread during GPU reset */
- mutex_lock(&adev->lock_reset);
+ r = down_read_killable(&adev->reset_sem);
+ if (r)
+ goto pro_end;
/* stop the scheduler */
kthread_park(ring->sched.thread);
@@ -1500,13 +1508,14 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
/* restart the scheduler */ kthread_unpark(ring->sched.thread);
- mutex_unlock(&adev->lock_reset);
+ up_read(&adev->reset_sem);
ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+pro_end:
kfree(fences);
- return 0;
+ return r;
}
static int amdgpu_debugfs_sclk_set(void *data, u64 val) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 78fd2c9a7b7d..82242e2f5658 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -2
@@ -3054,7 +3054,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,mutex_init (&adev->virt.vf_errors.lock);hash_init (adev->mn_hash);atomic_set (&adev->in_gpu_reset,0 ); -mutex_init (&adev->lock_reset); +init_rwsem (&adev->reset_sem);mutex_init (&adev->psp.mutex);mutex_init (&adev->notifier_lock);
@@ -4206,7 +4206,7 @@ static bool amdgpu_device_lock_adev(struct amdgpu_device *adev)if (atomic_cmpxchg (&adev->in_gpu_reset,0 ,1 ) !=0 )return false; -mutex_lock (&adev->lock_reset); +down_write (&adev->reset_sem);atomic_inc (&adev->gpu_reset_counter); switch (amdgpu_asic_reset_method(adev)) { @@ -4229,7 +4229,7 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
amdgpu_vf_error_trans_all(adev);
adev->mp1_state = PP_MP1_STATE_NONE;
atomic_set(&adev->in_gpu_reset, 0);
- mutex_unlock(&adev->lock_reset);
+ up_write(&adev->reset_sem);
}
static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index f27d83f2de78..8ac63f13fc6f 100644
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -8
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);int timeout =AI_MAILBOX_POLL_FLR_TIMEDOUT ; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ - locked =mutex_trylock (&adev->lock_reset); -if (locked) -atomic_set (&adev->in_gpu_reset,1 ); +down_read (&adev->reset_sem); do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -261,10 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
} while (timeout > 1);
flr_done:
- if (locked) {
- atomic_set(&adev->in_gpu_reset, 0);
- mutex_unlock(&adev->lock_reset);
- }
+ up_read(&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +1 -8
@@ -259,19 +259,12 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);int timeout =NV_MAILBOX_POLL_FLR_TIMEDOUT ; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ - locked =mutex_trylock (&adev->lock_reset); -if (locked) -atomic_set (&adev->in_gpu_reset,1 ); +down_read (&adev->reset_sem); do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -282,10 +275,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
} while (timeout > 1);
flr_done:
- if (locked) {
- atomic_set(&adev->in_gpu_reset, 0);
- mutex_unlock(&adev->lock_reset);
- }
+ up_read(&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
--
2.17.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -8
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);int timeout =AI_MAILBOX_POLL_FLR_TIMEDOUT ; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked =mutex_trylock (&adev->lock_reset); -if (locked) -atomic_set (&adev->in_gpu_reset,1 ); +down_read (&adev->reset_sem);
Above piece looks suspicious :
The original logic (before this patch and your another patch) is :
260 locked = mutex_trylock(&adev->lock_reset);
261 if (!locked)
262 return;
263
264 adev->in_gpu_reset = true;
So we only continue after the trylock success, and we "return" immediately upon the trylock fail,
With your change the code path continue anyway (did you change the logic in your another patch recently ??)
Please modify it as:
Locked = down_read_trylock(&adev->reset_sem);
If (!locked)
Return;
atomic_set(&adev->in_gpu_reset, 1);
_____________________________________
Monk Liu|GPU Virtualization Team |AMD
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Dennis Li
Sent: Thursday, August 20, 2020 5:33 PM
To: [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Cc: Li, Dennis <[email protected]>
Subject: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
clients don't need reset-lock for synchronization when no GPU recovery.
v2:
change to return the return value of down_read_killable.
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
@@ -954,7 +954,7 @@ struct amdgpu_device {atomic_t in_gpu_reset;enum pp_mp1_state mp1_state; -struct mutex lock_reset; +struct rw_semaphore reset_sem;struct amdgpu_doorbell_index doorbell_index;struct mutexnotifier_lock ;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -3
@@ -101,14 +101,18 @@ static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file) file->private_data = adev; -mutex_lock (&adev->lock_reset); +ret =down_read_killable (&adev->reset_sem); +if (ret) +return ret; +if (adev->autodump.dumping.done) {reinit_completion (&adev->autodump.dumping); ret =0 ; }else { ret = -EBUSY ; } -mutex_unlock (&adev->lock_reset); + +up_read (&adev->reset_sem);return ret; }
@@ -1242,7 +1246,9 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) }/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock (&adev->lock_reset); +r =down_read_killable (&adev->reset_sem); +if (r) +return r;/* hold on the scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; i++) { @@ -1269,7 +1275,7 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
kthread_unpark(ring->sched.thread);
}
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev);
@@ -1459,7 +1465,9 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
return -ENOMEM;
/* Avoid accidently unparking the sched thread during GPU reset */
-mutex_lock(&adev->lock_reset);
+r = down_read_killable(&adev->reset_sem);
+if (r)
+goto pro_end;
/* stop the scheduler */
kthread_park(ring->sched.thread);
@@ -1500,13 +1508,14 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
/* restart the scheduler */
kthread_unpark(ring->sched.thread);
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+pro_end:
kfree(fences);
-return 0;
+return r;
}
static int amdgpu_debugfs_sclk_set(void *data, u64 val) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 78fd2c9a7b7d..82242e2f5658 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -2
@@ -3054,7 +3054,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,mutex_init (&adev->virt.vf_errors.lock);hash_init (adev->mn_hash);atomic_set (&adev->in_gpu_reset,0 ); -mutex_init (&adev->lock_reset); +init_rwsem (&adev->reset_sem);mutex_init (&adev->psp.mutex);mutex_init (&adev->notifier_lock);
@@ -4206,7 +4206,7 @@ static bool amdgpu_device_lock_adev(struct amdgpu_device *adev)if (atomic_cmpxchg (&adev->in_gpu_reset,0 ,1 ) !=0 )return false; -mutex_lock (&adev->lock_reset); +down_write (&adev->reset_sem);atomic_inc (&adev->gpu_reset_counter); switch (amdgpu_asic_reset_method(adev)) { @@ -4229,7 +4229,7 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
amdgpu_vf_error_trans_all(adev);
adev->mp1_state = PP_MP1_STATE_NONE;
atomic_set(&adev->in_gpu_reset, 0);
-mutex_unlock(&adev->lock_reset);
+up_write(&adev->reset_sem);
}
static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index f27d83f2de78..8ac63f13fc6f 100644
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -8
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);int timeout =AI_MAILBOX_POLL_FLR_TIMEDOUT ; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked =mutex_trylock (&adev->lock_reset); -if (locked) -atomic_set (&adev->in_gpu_reset,1 ); +down_read (&adev->reset_sem); do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -261,10 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
} while (timeout > 1);
flr_done:
-if (locked) {
-atomic_set(&adev->in_gpu_reset, 0);
-mutex_unlock(&adev->lock_reset);
-}
+up_read(&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +1 -8
@@ -259,19 +259,12 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);int timeout =NV_MAILBOX_POLL_FLR_TIMEDOUT ; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked =mutex_trylock (&adev->lock_reset); -if (locked) -atomic_set (&adev->in_gpu_reset,1 ); +down_read (&adev->reset_sem); do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -282,10 +275,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
} while (timeout > 1);
flr_done:
-if (locked) {
-atomic_set(&adev->in_gpu_reset, 0);
-mutex_unlock(&adev->lock_reset);
-}
+up_read(&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
--
2.17.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7Cmonk.liu%40amd.com%7C83ea9660ce0749dbc2c208d844ec062c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637335127818823207&sdata=LR%2BvbRaX1Jonb%2F9wIvoK8gV%2FoTBLPeornTaQe9qS5xE%3D&reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
[AMD Official Use Only - Internal Distribution Only]
Hi, Monk,
See my below comments.
Best Regards
Dennis Li
[AMD Official Use Only - Internal Distribution Only]
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -5
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);
Quoted 10 lines
Quoted 9 lines
Above piece looks suspicious :
The original logic (before this patch and your another patch) is :
260 locked =
mutex_trylock(&adev->lock_reset);
261 if (!locked)
262 return;
263
264adev->in_gpu_reset= true;
Quoted 3 lines
Quoted 3 lines
So we only continue after the trylock success, and we "return" immediately upon the trylock fail,
With your change the code path continue anyway (did you change the logic in your another patch recently ??)
[Dennis Li] I didn't change the logic before, I guess that your local branch is different from drm-next. In drm-next, the logic is:
locked = mutex_trylock(&adev->lock_reset);
if (locked)atomic_set(&adev->in_gpu_reset, 1);
According to the comments before these codes, it wanted to block amdgpu_gpu_recover till msg FLR COMPLETE received, so I changed it in this patch.
Quoted 6 lines
Quoted 6 lines
Please modify it as:
Locked =
down_read_trylock(&adev->reset_sem);
If (!locked)
Return;atomic_set(&adev->in_gpu_reset, 1);
[Dennis Li] why need we set adev->in_gpu_reset as 1 here? It should be set when do GPU recovery.
_____________________________________
Monk Liu|GPU Virtualization Team |AMD
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Dennis Li
Sent: Thursday, August 20, 2020 5:33 PM
To: [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Cc: Li, Dennis <[email protected]>
Subject: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
clients don't need reset-lock for synchronization when no GPU recovery.
v2:
change to return the return value of down_read_killable.
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
@@ -954,7 +954,7 @@ struct amdgpu_device {atomic_t in_gpu_reset;enum pp_mp1_state mp1_state; -struct mutex lock_reset; +struct rw_semaphore reset_sem;struct amdgpu_doorbell_index doorbell_index;struct mutexnotifier_lock ;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -2
@@ -101,14 +101,18 @@ static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file) file->private_data = adev; -mutex_lock (&adev->lock_reset); +ret =down_read_killable (&adev->reset_sem); +if (ret) +return ret; +if (adev->autodump.dumping.done) {reinit_completion (&adev->autodump.dumping); ret =0 ; }else { ret = -EBUSY ; } -mutex_unlock (&adev->lock_reset); + +up_read (&adev->reset_sem);return ret; }
@@ -1242,7 +1246,9 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) }/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock (&adev->lock_reset); +r =down_read_killable (&adev->reset_sem); +if (r) +return r;/* hold on the scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; i++) { @@ -1269,7 +1275,7 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) kthread_unpark(ring->sched.thread); }
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
pm_runtime_mark_last_busy(dev->dev);
pm_runtime_put_autosuspend(dev->dev);
@@ -1459,7 +1465,9 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) return -ENOMEM;
/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock(&adev->lock_reset);
+r = down_read_killable(&adev->reset_sem);
+if (r)
+goto pro_end;
/* stop the scheduler */
kthread_park(ring->sched.thread);
@@ -1500,13 +1508,14 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
/* restart the scheduler */
kthread_unpark(ring->sched.thread);
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+pro_end:
kfree(fences);
-return 0;
+return r;
}
static int amdgpu_debugfs_sclk_set(void *data, u64 val) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 78fd2c9a7b7d..82242e2f5658 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -2
@@ -3054,7 +3054,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->virt.vf_errors.lock);hash_init (adev->mn_hash);atomic_set (&adev->in_gpu_reset,0 ); -mutex_init (&adev->lock_reset); +init_rwsem (&adev->reset_sem);mutex_init (&adev->psp.mutex);mutex_init (&adev->notifier_lock);
@@ -4206,7 +4206,7 @@ static bool amdgpu_device_lock_adev(struct amdgpu_device *adev) if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0) return false; -mutex_lock (&adev->lock_reset); +down_write (&adev->reset_sem);atomic_inc (&adev->gpu_reset_counter); switch (amdgpu_asic_reset_method(adev)) { @@ -4229,7 +4229,7 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) amdgpu_vf_error_trans_all(adev); adev->mp1_state = PP_MP1_STATE_NONE; atomic_set(&adev->in_gpu_reset, 0); -mutex_unlock(&adev->lock_reset); +up_write (&adev->reset_sem); }
static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index f27d83f2de78..8ac63f13fc6f 100644
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -9
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -261,10 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); -} +up_read (&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +2 -9
@@ -259,19 +259,12 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -282,10 +275,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); -} +up_read (&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
--
2.17.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7Cmonk.liu%40amd.com%7C83ea9660ce0749dbc2c208d844ec062c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637335127818823207&sdata=LR%2BvbRaX1Jonb%2F9wIvoK8gV%2FoTBLPeornTaQe9qS5xE%3D&reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
Quoted 4 lines
Quoted 4 lines
Locked =
down_read_trylock(&adev->reset_sem);
If (!locked)
Return;atomic_set(&adev->in_gpu_reset, 1);
[Dennis Li] why need we set adev->in_gpu_reset as 1 here? It should be set when do GPU recovery.
[ML] because "in_gpu_reset" means GPU is under reset or FLR (VF FLR actually)
If we get the reset_sem in flr_work routine, that means Host side is doing the VF FLR (flr_work is initiated from host side through an interrupt to guest)
Since host side is doing VF FLR thus we want to occupy GPU with 1) take the reset_sem first to prevent guest side GPU recovery routine occupy GPU; 2) mark GPU under reset by set "in_gpu_rest" to true
Thanks
_____________________________________
Monk Liu|GPU Virtualization Team |AMD
-----Original Message-----
From: Li, Dennis <[email protected]>
Sent: Thursday, August 20, 2020 7:36 PM
To: Liu, Monk <[email protected]>; [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Subject: RE: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
[AMD Official Use Only - Internal Distribution Only]
[AMD Official Use Only - Internal Distribution Only]
Hi, Monk,
See my below comments.
Best Regards
Dennis Li
[AMD Official Use Only - Internal Distribution Only]
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -5
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);
Quoted 10 lines
Quoted 9 lines
Above piece looks suspicious :
The original logic (before this patch and your another patch) is :
260 locked =
mutex_trylock(&adev->lock_reset);
261 if (!locked)
262 return;
263
264adev->in_gpu_reset= true;
Quoted 5 lines
Quoted 5 lines
So we only continue after the trylock success, and we "return" immediately upon the trylock fail,
With your change the code path continue anyway (did you change the logic in your another patch recently ??)
[Dennis Li] I didn't change the logic before, I guess that your local branch is different from drm-next. In drm-next, the logic is:
locked = mutex_trylock(&adev->lock_reset); if (locked) atomic_set(&adev->in_gpu_reset, 1); According to the comments before these codes, it wanted to block amdgpu_gpu_recover till msg FLR COMPLETE received, so I changed it in this patch.
Quoted 6 lines
Quoted 6 lines
Please modify it as:
Locked =
down_read_trylock(&adev->reset_sem);
If (!locked)
Return;atomic_set(&adev->in_gpu_reset, 1);
[Dennis Li] why need we set adev->in_gpu_reset as 1 here? It should be set when do GPU recovery.
_____________________________________
Monk Liu|GPU Virtualization Team |AMD
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Dennis Li
Sent: Thursday, August 20, 2020 5:33 PM
To: [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Cc: Li, Dennis <[email protected]>
Subject: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
clients don't need reset-lock for synchronization when no GPU recovery.
v2:
change to return the return value of down_read_killable.
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
@@ -954,7 +954,7 @@ struct amdgpu_device {atomic_t in_gpu_reset;enum pp_mp1_state mp1_state; -struct mutex lock_reset; +struct rw_semaphore reset_sem;struct amdgpu_doorbell_index doorbell_index;struct mutexnotifier_lock ;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -2
@@ -101,14 +101,18 @@ static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file) file->private_data = adev; -mutex_lock (&adev->lock_reset); +ret =down_read_killable (&adev->reset_sem); +if (ret) +return ret; +if (adev->autodump.dumping.done) {reinit_completion (&adev->autodump.dumping); ret =0 ; }else { ret = -EBUSY ; } -mutex_unlock (&adev->lock_reset); + +up_read (&adev->reset_sem);return ret; }
@@ -1242,7 +1246,9 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) }/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock (&adev->lock_reset); +r =down_read_killable (&adev->reset_sem); +if (r) +return r;/* hold on the scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; i++) { @@ -1269,7 +1275,7 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) kthread_unpark(ring->sched.thread); }
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
pm_runtime_mark_last_busy(dev->dev);
pm_runtime_put_autosuspend(dev->dev);
@@ -1459,7 +1465,9 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) return -ENOMEM;
/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock(&adev->lock_reset);
+r = down_read_killable(&adev->reset_sem);
+if (r)
+goto pro_end;
/* stop the scheduler */
kthread_park(ring->sched.thread);
@@ -1500,13 +1508,14 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
/* restart the scheduler */
kthread_unpark(ring->sched.thread);
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+pro_end:
kfree(fences);
-return 0;
+return r;
}
static int amdgpu_debugfs_sclk_set(void *data, u64 val) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 78fd2c9a7b7d..82242e2f5658 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -2
@@ -3054,7 +3054,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->virt.vf_errors.lock);hash_init (adev->mn_hash);atomic_set (&adev->in_gpu_reset,0 ); -mutex_init (&adev->lock_reset); +init_rwsem (&adev->reset_sem);mutex_init (&adev->psp.mutex);mutex_init (&adev->notifier_lock);
@@ -4206,7 +4206,7 @@ static bool amdgpu_device_lock_adev(struct amdgpu_device *adev) if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0) return false; -mutex_lock (&adev->lock_reset); +down_write (&adev->reset_sem);atomic_inc (&adev->gpu_reset_counter); switch (amdgpu_asic_reset_method(adev)) { @@ -4229,7 +4229,7 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) amdgpu_vf_error_trans_all(adev); adev->mp1_state = PP_MP1_STATE_NONE; atomic_set(&adev->in_gpu_reset, 0); -mutex_unlock(&adev->lock_reset); +up_write (&adev->reset_sem); }
static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index f27d83f2de78..8ac63f13fc6f 100644
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -9
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -261,10 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); -} +up_read (&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +2 -9
@@ -259,19 +259,12 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -282,10 +275,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); -} +up_read (&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
--
2.17.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7Cmonk.liu%40amd.com%7C83ea9660ce0749dbc2c208d844ec062c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637335127818823207&sdata=LR%2BvbRaX1Jonb%2F9wIvoK8gV%2FoTBLPeornTaQe9qS5xE%3D&reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
[AMD Official Use Only - Internal Distribution Only]
Hi, Monk,
Got it, thanks for your explanation.
Best Regards
Dennis Li
-----Original Message-----
From: Liu, Monk <[email protected]>
Sent: Friday, August 21, 2020 11:14 AM
To: Li, Dennis <[email protected]>; [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Subject: RE: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
[AMD Official Use Only - Internal Distribution Only]
Quoted 4 lines
Quoted 4 lines
Locked =
down_read_trylock(&adev->reset_sem);
If (!locked)
Return;atomic_set(&adev->in_gpu_reset, 1);
[Dennis Li] why need we set adev->in_gpu_reset as 1 here? It should be set when do GPU recovery.
[ML] because "in_gpu_reset" means GPU is under reset or FLR (VF FLR actually)
If we get the reset_sem in flr_work routine, that means Host side is doing the VF FLR (flr_work is initiated from host side through an interrupt to guest)
Since host side is doing VF FLR thus we want to occupy GPU with 1) take the reset_sem first to prevent guest side GPU recovery routine occupy GPU; 2) mark GPU under reset by set "in_gpu_rest" to true
Thanks
_____________________________________
Monk Liu|GPU Virtualization Team |AMD
-----Original Message-----
From: Li, Dennis <[email protected]>
Sent: Thursday, August 20, 2020 7:36 PM
To: Liu, Monk <[email protected]>; [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Subject: RE: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
[AMD Official Use Only - Internal Distribution Only]
[AMD Official Use Only - Internal Distribution Only]
Hi, Monk,
See my below comments.
Best Regards
Dennis Li
[AMD Official Use Only - Internal Distribution Only]
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -5
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);
Quoted 10 lines
Quoted 9 lines
Above piece looks suspicious :
The original logic (before this patch and your another patch) is :
260 locked =
mutex_trylock(&adev->lock_reset);
261 if (!locked)
262 return;
263
264adev->in_gpu_reset= true;
Quoted 5 lines
Quoted 5 lines
So we only continue after the trylock success, and we "return" immediately upon the trylock fail,
With your change the code path continue anyway (did you change the logic in your another patch recently ??)
[Dennis Li] I didn't change the logic before, I guess that your local branch is different from drm-next. In drm-next, the logic is:
locked = mutex_trylock(&adev->lock_reset); if (locked) atomic_set(&adev->in_gpu_reset, 1); According to the comments before these codes, it wanted to block amdgpu_gpu_recover till msg FLR COMPLETE received, so I changed it in this patch.
Quoted 6 lines
Quoted 6 lines
Please modify it as:
Locked =
down_read_trylock(&adev->reset_sem);
If (!locked)
Return;atomic_set(&adev->in_gpu_reset, 1);
[Dennis Li] why need we set adev->in_gpu_reset as 1 here? It should be set when do GPU recovery.
_____________________________________
Monk Liu|GPU Virtualization Team |AMD
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Dennis Li
Sent: Thursday, August 20, 2020 5:33 PM
To: [email protected]; Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>; Zhang, Hawking <[email protected]>; Koenig, Christian <[email protected]>
Cc: Li, Dennis <[email protected]>
Subject: [PATCH v2] drm/amdgpu: change reset lock from mutex to rw_semaphore
clients don't need reset-lock for synchronization when no GPU recovery.
v2:
change to return the return value of down_read_killable.
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
@@ -954,7 +954,7 @@ struct amdgpu_device {atomic_t in_gpu_reset;enum pp_mp1_state mp1_state; -struct mutex lock_reset; +struct rw_semaphore reset_sem;struct amdgpu_doorbell_index doorbell_index;struct mutexnotifier_lock ;
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +9 -2
@@ -101,14 +101,18 @@ static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file) file->private_data = adev; -mutex_lock (&adev->lock_reset); +ret =down_read_killable (&adev->reset_sem); +if (ret) +return ret; +if (adev->autodump.dumping.done) {reinit_completion (&adev->autodump.dumping); ret =0 ; }else { ret = -EBUSY ; } -mutex_unlock (&adev->lock_reset); + +up_read (&adev->reset_sem);return ret; }
@@ -1242,7 +1246,9 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) }/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock (&adev->lock_reset); +r =down_read_killable (&adev->reset_sem); +if (r) +return r;/* hold on the scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; i++) { @@ -1269,7 +1275,7 @@ static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) kthread_unpark(ring->sched.thread); }
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
pm_runtime_mark_last_busy(dev->dev);
pm_runtime_put_autosuspend(dev->dev);
@@ -1459,7 +1465,9 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) return -ENOMEM;
/* Avoid accidently unparking the sched thread during GPU reset */ -mutex_lock(&adev->lock_reset);
+r = down_read_killable(&adev->reset_sem);
+if (r)
+goto pro_end;
/* stop the scheduler */
kthread_park(ring->sched.thread);
@@ -1500,13 +1508,14 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
/* restart the scheduler */
kthread_unpark(ring->sched.thread);
-mutex_unlock(&adev->lock_reset);
+up_read(&adev->reset_sem);
ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+pro_end:
kfree(fences);
-return 0;
+return r;
}
static int amdgpu_debugfs_sclk_set(void *data, u64 val) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 78fd2c9a7b7d..82242e2f5658 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -2
@@ -3054,7 +3054,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->virt.vf_errors.lock);hash_init (adev->mn_hash);atomic_set (&adev->in_gpu_reset,0 ); -mutex_init (&adev->lock_reset); +init_rwsem (&adev->reset_sem);mutex_init (&adev->psp.mutex);mutex_init (&adev->notifier_lock);
@@ -4206,7 +4206,7 @@ static bool amdgpu_device_lock_adev(struct amdgpu_device *adev) if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0) return false; -mutex_lock (&adev->lock_reset); +down_write (&adev->reset_sem);atomic_inc (&adev->gpu_reset_counter); switch (amdgpu_asic_reset_method(adev)) { @@ -4229,7 +4229,7 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) amdgpu_vf_error_trans_all(adev); adev->mp1_state = PP_MP1_STATE_NONE; atomic_set(&adev->in_gpu_reset, 0); -mutex_unlock(&adev->lock_reset); +up_write (&adev->reset_sem); }
static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index f27d83f2de78..8ac63f13fc6f 100644
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -9
@@ -238,19 +238,12 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);do { if (xgpu_ai_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -261,10 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); -} +up_read (&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +2 -9
@@ -259,19 +259,12 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT; -int locked;/* block amdgpu_gpu_recover till msg FLR COMPLETE received, * otherwise the mailbox msg will be ruined/reseted by * the VF FLR. -* -* we can unlock the lock_reset to allow "amdgpu_job_timedout" -* to run gpu_recover() after FLR_NOTIFICATION_CMPL received -* which means host side had finished this VF's FLR. */ -locked = mutex_trylock(&adev->lock_reset); -if (locked) -atomic_set(&adev->in_gpu_reset, 1); +down_read (&adev->reset_sem);do { if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL) @@ -282,10 +275,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) } while (timeout > 1); flr_done: -if (locked) { -atomic_set (&adev->in_gpu_reset,0 ); -mutex_unlock (&adev->lock_reset); -} +up_read (&adev->reset_sem);
/* Trigger recovery for world switch failure if no TDR */ if (amdgpu_device_should_recover_gpu(adev)
--
2.17.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7Cmonk.liu%40amd.com%7C83ea9660ce0749dbc2c208d844ec062c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637335127818823207&sdata=LR%2BvbRaX1Jonb%2F9wIvoK8gV%2FoTBLPeornTaQe9qS5xE%3D&reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx