Skip to contentSkip to conversation

drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

PATCH In review Yang Wang 5 participants 15 messages Apr 20, 2022 – Apr 21, 2022

Subsystems (3)
lore
Share & tools
lore

Yang Wang

author
#0 lore raw

PATCH 1/1 drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc() to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
 drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +11 -3
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
 static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
 {
 	ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-			GFP_KERNEL | __GFP_ZERO);
+				    GFP_KERNEL);
 	if (!ttm->pages)
 		return -ENOMEM;
+
+	memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
 	return 0;
 }
 
@@ -108,10 +111,12 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)
 	ttm->pages = kvmalloc_array(ttm->num_pages,
 				    sizeof(*ttm->pages) +
 				    sizeof(*ttm->dma_address),
-				    GFP_KERNEL | __GFP_ZERO);
+				    GFP_KERNEL);
 	if (!ttm->pages)
 		return -ENOMEM;
 
+	memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));
+
 	ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
 	return 0;
 }
@@ -120,9 +125,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)
 {
 	ttm->dma_address = kvmalloc_array(ttm->num_pages,
 					  sizeof(*ttm->dma_address),
-					  GFP_KERNEL | __GFP_ZERO);
+					  GFP_KERNEL);
 	if (!ttm->dma_address)
 		return -ENOMEM;
+
+	memset(ttm->dma_address, 0, ttm->num_pages * sizeof(*ttm->dma_address));
+
 	return 0;
 }
 
Signature · 1 line
2.25.1
#1 lore raw

Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

Quoted 56 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
  drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
  1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +11 -3
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
  static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
  	ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-			GFP_KERNEL | __GFP_ZERO);
+				    GFP_KERNEL);
  	if (!ttm->pages)
  		return -ENOMEM;
+
+	memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
  	return 0;
  }
 
@@ -108,10 +111,12 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)
  	ttm->pages = kvmalloc_array(ttm->num_pages,
  				    sizeof(*ttm->pages) +
  				    sizeof(*ttm->dma_address),
-				    GFP_KERNEL | __GFP_ZERO);
+				    GFP_KERNEL);
  	if (!ttm->pages)
  		return -ENOMEM;
 
+	memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));
+
  	ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
  	return 0;
  }
@@ -120,9 +125,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
  	ttm->dma_address = kvmalloc_array(ttm->num_pages,
  					  sizeof(*ttm->dma_address),
-					  GFP_KERNEL | __GFP_ZERO);
+					  GFP_KERNEL);
  	if (!ttm->dma_address)
  		return -ENOMEM;
+
+	memset(ttm->dma_address, 0, ttm->num_pages * sizeof(*ttm->dma_address));
+
  	return 0;
  }
 
#2 lore raw

[AMD Official Use Only]

________________________________
From: Koenig, Christian <[email protected]>
Sent: Wednesday, April 20, 2022 5:00 PM
To: Wang, Yang(Kevin) <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail. it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...

        if ((flags & GFP_KERNEL) != GFP_KERNEL)
                return kmalloc_node(size, flags, node);

...
}

Best Regards,
Kevin

Quoted 56 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
  drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
  1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +11 -3
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
  static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
       return 0;
  }
 
@@ -108,10 +111,12 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)
       ttm->pages = kvmalloc_array(ttm->num_pages,
                                   sizeof(*ttm->pages) +
                                   sizeof(*ttm->dma_address),
-                                 GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
 
+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));
+
       ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
       return 0;
  }
@@ -120,9 +125,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                         sizeof(*ttm->dma_address),
-                                       GFP_KERNEL | __GFP_ZERO);
+                                       GFP_KERNEL);
       if (!ttm->dma_address)
               return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * sizeof(*ttm->dma_address));
+
       return 0;
  }
 
#3 lore raw

Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 16 lines

[AMD Official Use Only]

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]>
*Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

Quoted 7 lines

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!

The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 75 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...
      if ((flags & GFP_KERNEL) != GFP_KERNEL) return kmalloc_node(size, flags, node);
...
}

Best Regards,
Kevin

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object *bo,

Quoted 15 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
   drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---

   1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, 

bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
    static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
    {
         ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
         if (!ttm->pages)
                 return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
         return 0;
    }
 
@@ -108,10 +111,12 @@ static int 

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
         ttm->pages = kvmalloc_array(ttm->num_pages,

sizeof(*ttm->pages) +
sizeof(*ttm->dma_address),
-                                 GFP_KERNEL | __GFP_ZERO); +                                 GFP_KERNEL);
        if (!ttm->pages)
                return -ENOMEM;

+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) +

sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines

+
        ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); return 0;
   }

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -120,9 +125,12 @@ static int 

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
    {
         ttm->dma_address = kvmalloc_array(ttm->num_pages,
 sizeof(*ttm->dma_address),
-                                       GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->dma_address)
                 return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * 

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
         return 0;
    }
 
#4 lore raw

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree<https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v5.18-rc3&id=a421ef303008b0ceee2cfc625c3246fa7654b0ca>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

________________________________
From: Koenig, Christian <[email protected]>
Sent: Wednesday, April 20, 2022 6:53 PM
To: Wang, Yang(Kevin) <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 5:00 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!

The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...

        if ((flags & GFP_KERNEL) != GFP_KERNEL)
                return kmalloc_node(size, flags, node);

...
}

Best Regards,
Kevin

Quoted 56 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
  drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
  1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +11 -3
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
  static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
       return 0;
  }
 
@@ -108,10 +111,12 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)
       ttm->pages = kvmalloc_array(ttm->num_pages,
                                   sizeof(*ttm->pages) +
                                   sizeof(*ttm->dma_address),
-                                 GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
 
+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));
+
       ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
       return 0;
  }
@@ -120,9 +125,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                         sizeof(*ttm->dma_address),
-                                       GFP_KERNEL | __GFP_ZERO);
+                                       GFP_KERNEL);
       if (!ttm->dma_address)
               return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * sizeof(*ttm->dma_address));
+
       return 0;
  }
 
#5 lore raw

Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

Quoted 141 lines

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v5.18-rc3&id=a421ef303008b0ceee2cfc625c3246fa7654b0ca>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]>
*Sent:* Wednesday, April 20, 2022 6:53 PM
*To:* Wang, Yang(Kevin) <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 17 lines

[AMD Official Use Only]

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use

vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

Quoted 7 lines

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!
*
*The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 77 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...
        if ((flags & GFP_KERNEL) != GFP_KERNEL) return kmalloc_node(size, flags, node);
...
}

Best Regards,
Kevin

Quoted 6 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

<mailto:[email protected]>

Review

Quoted 5 lines

---
   drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---

   1 file changed, 11 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/drivers/gpu/drm/ttm/ttm_tt.c

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object *bo,

Quoted 4 lines

index 79c870a3bef8..9f2f3e576b8d 100644

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, 

bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
    static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
    {
         ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->pages)
                 return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
         return 0;
    }
 
@@ -108,10 +111,12 @@ static int 

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
         ttm->pages = kvmalloc_array(ttm->num_pages,

sizeof(*ttm->pages) +
sizeof(*ttm->dma_address),
-                                 GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
        if (!ttm->pages)
                return -ENOMEM;

+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) +

sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines

+
        ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); return 0;
   }

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -120,9 +125,12 @@ static int 

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
    {
         ttm->dma_address = kvmalloc_array(ttm->num_pages,
 sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->dma_address)
                 return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * 

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
         return 0;
    }
 
#6 lore raw

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM) bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

        initial_domain = (u32)(0xffffffff & args->in.domains);
retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment,
                                     initial_domain,
                                     flags, ttm_bo_type_device, resv, &gobj);
        if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
                        flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
                        goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
                        initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
                        goto retry;
                }
                DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
                                size, initial_domain, args->in.alignment, r);
        }

Best Regards,
Kevin

________________________________
From: Christian König <[email protected]>
Sent: Wednesday, April 20, 2022 7:55 PM
To: Wang, Yang(Kevin) <[email protected]>; Koenig, Christian <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit.kernel.org%2Fpub%2Fscm%2Flinux%2Fkernel%2Fgit%2Ftorvalds%2Flinux.git%2Fcommit%2F%3Fh%3Dv5.18-rc3%26id%3Da421ef303008b0ceee2cfc625c3246fa7654b0ca&data=05%7C01%7CKevinYang.Wang%40amd.com%7C2e9fd86a27d64a39432508da22c4b1f1%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637860525454702938%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=CiP9x3grwQ2aXFZPjpsAtwLCpBVeJufbGngy%2BtXLFbM%3D&reserved=0>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 6:53 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 5:00 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!

The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...

        if ((flags & GFP_KERNEL) != GFP_KERNEL)
                return kmalloc_node(size, flags, node);

...
}

Best Regards,
Kevin

Quoted 56 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
  drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
  1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +11 -3
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
  static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
       return 0;
  }
 
@@ -108,10 +111,12 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)
       ttm->pages = kvmalloc_array(ttm->num_pages,
                                   sizeof(*ttm->pages) +
                                   sizeof(*ttm->dma_address),
-                                 GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
 
+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));
+
       ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
       return 0;
  }
@@ -120,9 +125,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                         sizeof(*ttm->dma_address),
-                                       GFP_KERNEL | __GFP_ZERO);
+                                       GFP_KERNEL);
       if (!ttm->dma_address)
               return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * sizeof(*ttm->dma_address));
+
       return 0;
  }
 
#7 lore raw

Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

Quoted 195 lines

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM)  bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

  initial_domain = (u32)(0xffffffff & args->in.domains); retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment, initial_domain,
                 flags, ttm_bo_type_device, resv, &gobj); if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
    goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
    goto retry;
                }
DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", size, initial_domain, args->in.alignment, r); }

Best Regards,
Kevin

------------------------------------------------------------------------
*From:* Christian König <[email protected]>
*Sent:* Wednesday, April 20, 2022 7:55 PM
*To:* Wang, Yang(Kevin) <[email protected]>; Koenig, Christian <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

Quoted 142 lines

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree <https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit.kernel.org%2Fpub%2Fscm%2Flinux%2Fkernel%2Fgit%2Ftorvalds%2Flinux.git%2Fcommit%2F%3Fh%3Dv5.18-rc3%26id%3Da421ef303008b0ceee2cfc625c3246fa7654b0ca&data=05%7C01%7CKevinYang.Wang%40amd.com%7C2e9fd86a27d64a39432508da22c4b1f1%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637860525454702938%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=CiP9x3grwQ2aXFZPjpsAtwLCpBVeJufbGngy%2BtXLFbM%3D&reserved=0>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 6:53 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 17 lines

[AMD Official Use Only]

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use

vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

Quoted 7 lines

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!
*
*The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 78 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...
if ((flags & GFP_KERNEL) != GFP_KERNEL)
        return kmalloc_node(size, flags, node);
...
}

Best Regards,
Kevin

Quoted 1 line

to allocate memory, when request size is exceeds kmalloc limit, it

will

Quoted 5 lines

cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

<mailto:[email protected]>

Review

Quoted 5 lines

---
   drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---

   1 file changed, 11 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/drivers/gpu/drm/ttm/ttm_tt.c

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object *bo,

Quoted 4 lines

index 79c870a3bef8..9f2f3e576b8d 100644

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, 

bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
    static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
    {
         ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->pages)
                 return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
         return 0;
    }
 
@@ -108,10 +111,12 @@ static int 

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
         ttm->pages = kvmalloc_array(ttm->num_pages,

sizeof(*ttm->pages) +
sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
        if (!ttm->pages)
                return -ENOMEM;

+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages)

+ sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines

+
        ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); return 0;
   }

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -120,9 +125,12 @@ static int 

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int · line 132

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
    {
         ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                        sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->dma_address)
                 return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * 

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
         return 0;
    }
 
#8 lore raw

[AMD Official Use Only]

Hi Chris,

1) Change the test case to use something larger than 1TiB.
sure, we can increase the size of BO and make test pass,
but if user really want to allocate 1TB GTT BO, we have no reason to let it fail? right? the system availed memory about 2T, but it will still fail.

2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path. the 5.18 kernel will add this patch to fix this issue .

Best Regards,
Kevin
________________________________
From: Koenig, Christian <[email protected]>
Sent: Wednesday, April 20, 2022 8:42 PM
To: Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM) bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

        initial_domain = (u32)(0xffffffff & args->in.domains);
retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment,
                                     initial_domain,
                                     flags, ttm_bo_type_device, resv, &gobj);
        if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
                        flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
                        goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
                        initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
                        goto retry;
                }
                DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
                                size, initial_domain, args->in.alignment, r);
        }

Best Regards,
Kevin

________________________________
From: Christian König <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 7:55 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; Koenig, Christian <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit.kernel.org%2Fpub%2Fscm%2Flinux%2Fkernel%2Fgit%2Ftorvalds%2Flinux.git%2Fcommit%2F%3Fh%3Dv5.18-rc3%26id%3Da421ef303008b0ceee2cfc625c3246fa7654b0ca&data=05%7C01%7CKevinYang.Wang%40amd.com%7C2e9fd86a27d64a39432508da22c4b1f1%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637860525454702938%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=CiP9x3grwQ2aXFZPjpsAtwLCpBVeJufbGngy%2BtXLFbM%3D&reserved=0>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 6:53 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 5:00 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!

The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...

        if ((flags & GFP_KERNEL) != GFP_KERNEL)
                return kmalloc_node(size, flags, node);

...
}

Best Regards,
Kevin

Quoted 56 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
  drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
  1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +11 -3
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
  static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
       return 0;
  }
 
@@ -108,10 +111,12 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)
       ttm->pages = kvmalloc_array(ttm->num_pages,
                                   sizeof(*ttm->pages) +
                                   sizeof(*ttm->dma_address),
-                                 GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
 
+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));
+
       ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
       return 0;
  }
@@ -120,9 +125,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                         sizeof(*ttm->dma_address),
-                                       GFP_KERNEL | __GFP_ZERO);
+                                       GFP_KERNEL);
       if (!ttm->dma_address)
               return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * sizeof(*ttm->dma_address));
+
       return 0;
  }
 
#9 lore raw

Am 20.04.22 um 14:54 schrieb Wang, Yang(Kevin):

Quoted 9 lines

[AMD Official Use Only]

Hi Chris,

1) Change the test case to use something larger than 1TiB.
sure, we can increase the size of BO and make test pass,
but if user really want to allocate 1TB GTT BO, we have no reason to let it fail? right?

No, the reason is the underlying core kernel doesn't allow kvmalloc allocations with GFP_ZERO which are large enough to hold the array of allocated pages for this.

We are working on top of the core Linux kernel and should *NEVER* ever add workarounds like what was suggested here.

Regards,
Christian.

Quoted 220 lines

the system availed memory about 2T, but it will still fail.

2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path. the 5.18 kernel will add this patch to fix this issue .

Best Regards,
Kevin
------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]>
*Sent:* Wednesday, April 20, 2022 8:42 PM
*To:* Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

Quoted 195 lines

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM)  bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

initial_domain = (u32)(0xffffffff & args->in.domains); retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment, initial_domain,
                   flags, ttm_bo_type_device, resv, &gobj); if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
      goto retry;
                }
DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", size, initial_domain, args->in.alignment, r); }

Best Regards,
Kevin

------------------------------------------------------------------------
*From:* Christian König <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 7:55 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; Koenig, Christian <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

Quoted 142 lines

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree <https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit.kernel.org%2Fpub%2Fscm%2Flinux%2Fkernel%2Fgit%2Ftorvalds%2Flinux.git%2Fcommit%2F%3Fh%3Dv5.18-rc3%26id%3Da421ef303008b0ceee2cfc625c3246fa7654b0ca&data=05%7C01%7CKevinYang.Wang%40amd.com%7C2e9fd86a27d64a39432508da22c4b1f1%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637860525454702938%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=CiP9x3grwQ2aXFZPjpsAtwLCpBVeJufbGngy%2BtXLFbM%3D&reserved=0>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 6:53 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 17 lines

[AMD Official Use Only]

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use

vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

Quoted 7 lines

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!
*
*The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 78 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...
      if ((flags & GFP_KERNEL) != GFP_KERNEL) return kmalloc_node(size, flags, node);
...
}

Best Regards,
Kevin

Quoted 1 line

to allocate memory, when request size is exceeds kmalloc limit,

it will

Quoted 5 lines

cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

<mailto:[email protected]>

Review

Quoted 5 lines

---
   drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---

   1 file changed, 11 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/drivers/gpu/drm/ttm/ttm_tt.c

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object

Quoted 4 lines

index 79c870a3bef8..9f2f3e576b8d 100644

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object 

*bo, bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
    static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
    {
         ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->pages)
                 return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
         return 0;
    }
 
@@ -108,10 +111,12 @@ static int 

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
         ttm->pages = kvmalloc_array(ttm->num_pages,

                                    sizeof(*ttm->pages) + sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
        if (!ttm->pages)
                return -ENOMEM;

+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages)

+ sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines

+
        ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); return 0;
   }

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -120,9 +125,12 @@ static int 

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int · line 132

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
    {
         ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                        sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->dma_address)
                 return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * 

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
         return 0;
    }
 
#10 lore raw

[AMD Official Use Only]

________________________________
From: Koenig, Christian <[email protected]>
Sent: Wednesday, April 20, 2022 8:56 PM
To: Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 14:54 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

Hi Chris,

1) Change the test case to use something larger than 1TiB.
sure, we can increase the size of BO and make test pass,
but if user really want to allocate 1TB GTT BO, we have no reason to let it fail? right?

No, the reason is the underlying core kernel doesn't allow kvmalloc allocations with GFP_ZERO which are large enough to hold the array of allocated pages for this.

We are working on top of the core Linux kernel and should *NEVER* ever add workarounds like what was suggested here.

Regards,
Christian.

Kevin:

it is ok to explain why we do not need fix this issue at this stage.

thanks.

Best Regards,
Kevin
the system availed memory about 2T, but it will still fail.

2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path. the 5.18 kernel will add this patch to fix this issue .

Best Regards,
Kevin
________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 8:42 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; Christian König <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM) bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

        initial_domain = (u32)(0xffffffff & args->in.domains);
retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment,
                                     initial_domain,
                                     flags, ttm_bo_type_device, resv, &gobj);
        if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
                        flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
                        goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
                        initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
                        goto retry;
                }
                DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
                                size, initial_domain, args->in.alignment, r);
        }

Best Regards,
Kevin

________________________________
From: Christian König <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 7:55 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; Koenig, Christian <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit.kernel.org%2Fpub%2Fscm%2Flinux%2Fkernel%2Fgit%2Ftorvalds%2Flinux.git%2Fcommit%2F%3Fh%3Dv5.18-rc3%26id%3Da421ef303008b0ceee2cfc625c3246fa7654b0ca&data=05%7C01%7CKevinYang.Wang%40amd.com%7C2e9fd86a27d64a39432508da22c4b1f1%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637860525454702938%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=CiP9x3grwQ2aXFZPjpsAtwLCpBVeJufbGngy%2BtXLFbM%3D&reserved=0>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 6:53 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

[AMD Official Use Only]

________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Wednesday, April 20, 2022 5:00 PM
To: Wang, Yang(Kevin) <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!

The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...

        if ((flags & GFP_KERNEL) != GFP_KERNEL)
                return kmalloc_node(size, flags, node);

...
}

Best Regards,
Kevin

Quoted 56 lines

to allocate memory, when request size is exceeds kmalloc limit, it will cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

---
  drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
  1 file changed, 11 insertions(+), 3 deletions(-)
drivers/gpu/drm/ttm/ttm_tt.c +11 -3
index 79c870a3bef8..9f2f3e576b8d 100644
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
  static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
       return 0;
  }
 
@@ -108,10 +111,12 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)
       ttm->pages = kvmalloc_array(ttm->num_pages,
                                   sizeof(*ttm->pages) +
                                   sizeof(*ttm->dma_address),
-                                 GFP_KERNEL | __GFP_ZERO);
+                                 GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
 
+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));
+
       ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
       return 0;
  }
@@ -120,9 +125,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                         sizeof(*ttm->dma_address),
-                                       GFP_KERNEL | __GFP_ZERO);
+                                       GFP_KERNEL);
       if (!ttm->dma_address)
               return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * sizeof(*ttm->dma_address));
+
       return 0;
  }
 
#11 lore raw
In reply to Christian König
Quoted 7 lines · Christian König

Am 20.04.22 um 14:54 schrieb Wang, Yang(Kevin):

Quoted 6 lines

[AMD Official Use Only]

Hi Chris,

Quoted 7 lines
Quoted 3 lines

1) Change the test case to use something larger than 1TiB.
sure, we can increase the size of BO and make test pass,
but if user really want to allocate 1TB GTT BO, we have no reason to let it fail? right?

No, the reason is the underlying core kernel doesn't allow kvmalloc allocations with GFP_ZERO which are large enough to hold the array of allocated pages for this.

We are working on top of the core Linux kernel and should *NEVER* ever add workarounds like what was suggested here. >

AFAIU, for the purpose of ttm use, fallback to vmalloc is fine.

* Please note that any use of gfp flags outside of GFP_KERNEL is
careful to not
* fall back to vmalloc.
*

Actually the current implementation documents the behavior, but it is deep inside the implementation to be noticeable - at least not obvious while using kvmalloc_array.

Thanks,
Lijo

Quoted 224 lines

Regards,
Christian.

Quoted 220 lines

the system availed memory about 2T, but it will still fail.

2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path. the 5.18 kernel will add this patch to fix this issue .

Best Regards,
Kevin
------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]>
*Sent:* Wednesday, April 20, 2022 8:42 PM
*To:* Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

Quoted 195 lines

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM)  bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

initial_domain = (u32)(0xffffffff & args->in.domains); retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment, initial_domain,
                   flags, ttm_bo_type_device, resv, &gobj); if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
      goto retry;
                }
DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", size, initial_domain, args->in.alignment, r); }

Best Regards,
Kevin

------------------------------------------------------------------------
*From:* Christian König <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 7:55 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; Koenig, Christian <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

Quoted 142 lines

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree <https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit.kernel.org%2Fpub%2Fscm%2Flinux%2Fkernel%2Fgit%2Ftorvalds%2Flinux.git%2Fcommit%2F%3Fh%3Dv5.18-rc3%26id%3Da421ef303008b0ceee2cfc625c3246fa7654b0ca&data=05%7C01%7Clijo.lazar%40amd.com%7C908fad4b756248e06e5e08da22cd4463%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637860562263637656%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=34hU%2FcxcRgBiiZ2jdNxTVmU7t4JF7TR27hx1b119U9g%3D&reserved=0>
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 6:53 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 17 lines

[AMD Official Use Only]

------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use

vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

Quoted 7 lines

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!
*
*The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 78 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...
      if ((flags & GFP_KERNEL) != GFP_KERNEL) return kmalloc_node(size, flags, node);
...
}

Best Regards,
Kevin

Quoted 1 line

to allocate memory, when request size is exceeds kmalloc limit,

it will

Quoted 5 lines

cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

<mailto:[email protected]>

Review

Quoted 5 lines

---
   drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---

   1 file changed, 11 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/drivers/gpu/drm/ttm/ttm_tt.c

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object

Quoted 4 lines

index 79c870a3bef8..9f2f3e576b8d 100644

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object 

*bo, bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
    static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
    {
         ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->pages)
                 return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
         return 0;
    }
 
@@ -108,10 +111,12 @@ static int 

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
         ttm->pages = kvmalloc_array(ttm->num_pages,

                                    sizeof(*ttm->pages) + sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
        if (!ttm->pages)
                return -ENOMEM;

+     memset(ttm->pages, 0, ttm->num_pages * (sizeof(*ttm->pages)

+ sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines

+
        ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); return 0;
   }

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -120,9 +125,12 @@ static int 

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int · line 132

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
    {
         ttm->dma_address = kvmalloc_array(ttm->num_pages,
                                        sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->dma_address)
                 return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * 

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
         return 0;
    }
 
#12 lore raw
Quoted 16 lines · Lijo
Quoted 13 lines · Christian König

Am 20.04.22 um 14:54 schrieb Wang, Yang(Kevin):

Quoted 9 lines

[AMD Official Use Only]

Hi Chris,

1) Change the test case to use something larger than 1TiB.
sure, we can increase the size of BO and make test pass,
but if user really want to allocate 1TB GTT BO, we have no reason to let it fail? right?

No, the reason is the underlying core kernel doesn't allow kvmalloc allocations with GFP_ZERO which are large enough to hold the array of allocated pages for this.

Quoted 7 lines
Quoted 1 line

We are working on top of the core Linux kernel and should *NEVER* ever add workarounds like what was suggested here. >

AFAIU, for the purpose of ttm use, fallback to vmalloc is fine.

 * Please note that any use of gfp flags outside of GFP_KERNEL is careful to not
 * fall back to vmalloc.
 *

That's weird, because kvcalloc does the same thing. If that were not able to fall back to vmalloc, it would be pretty useless.

   static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags)
   {
            return kvmalloc_array(n, size, flags | __GFP_ZERO);
   }

Maybe kvcalloc is the function we TTM should be using here anyway, instead of open-coding the kvmalloc_array call with an extra GFP flag.

Regards,
  Felix

Quoted 234 lines

Actually the current implementation documents the behavior, but it is deep inside the implementation to be noticeable - at least not obvious while using kvmalloc_array.

Thanks,
Lijo

Quoted 228 lines

Regards,
Christian.

Quoted 224 lines

the system availed memory about 2T, but it will still fail.

2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path. the 5.18 kernel will add this patch to fix this issue .

Best Regards,
Kevin
------------------------------------------------------------------------

*From:* Koenig, Christian <[email protected]>
*Sent:* Wednesday, April 20, 2022 8:42 PM
*To:* Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

Quoted 198 lines

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM)  bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

initial_domain = (u32)(0xffffffff & args->in.domains); retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment, initial_domain,
                   flags, ttm_bo_type_device, resv, &gobj); if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
      goto retry;
                }
DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", size, initial_domain, args->in.alignment, r); }

Best Regards,
Kevin

------------------------------------------------------------------------

*From:* Christian König <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 7:55 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; Koenig, Christian <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

Quoted 144 lines

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v5.18-rc3&id=a421ef303008b0ceee2cfc625c3246fa7654b0ca
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

------------------------------------------------------------------------

*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 6:53 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 18 lines

[AMD Official Use Only]

------------------------------------------------------------------------

*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use

vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

Quoted 7 lines

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!
*
*The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 78 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...
      if ((flags & GFP_KERNEL) != GFP_KERNEL) return kmalloc_node(size, flags, node);
...
}

Best Regards,
Kevin

Quoted 1 line

to allocate memory, when request size is exceeds kmalloc limit,

it will

Quoted 5 lines

cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

<mailto:[email protected]>

Review

Quoted 5 lines

---
   drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---

   1 file changed, 11 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/drivers/gpu/drm/ttm/ttm_tt.c

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object

Quoted 4 lines

index 79c870a3bef8..9f2f3e576b8d 100644

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object 

*bo, bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
    static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
    {
         ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->pages)
                 return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
         return 0;
    }
 
@@ -108,10 +111,12 @@ static int 

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
         ttm->pages = kvmalloc_array(ttm->num_pages,

sizeof(*ttm->pages) +
sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
        if (!ttm->pages)
                return -ENOMEM;

+     memset(ttm->pages, 0, ttm->num_pages *

(sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines

+
        ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); return 0;
   }

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -120,9 +125,12 @@ static int 

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int · line 132

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
    {
         ttm->dma_address = kvmalloc_array(ttm->num_pages,
 sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->dma_address)
                 return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages * 

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
         return 0;
    }
 
#13 lore raw

[AMD Official Use Only]

________________________________
From: Kuehling, Felix <[email protected]>
Sent: Thursday, April 21, 2022 5:21 AM
To: Lazar, Lijo <[email protected]>; Koenig, Christian <[email protected]>; Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Quoted 22 lines · Lijo
Quoted 12 lines · Christian König

Am 20.04.22 um 14:54 schrieb Wang, Yang(Kevin):

Quoted 10 lines

[AMD Official Use Only]

Hi Chris,

1) Change the test case to use something larger than 1TiB. sure, we can increase the size of BO and make test pass, but if user really want to allocate 1TB GTT BO, we have no reason to let it fail? right?

Quoted 6 lines

No, the reason is the underlying core kernel doesn't allow kvmalloc allocations with GFP_ZERO which are large enough to hold the array of allocated pages for this.

We are working on top of the core Linux kernel and should *NEVER* ever add workarounds like what was suggested here. >

Quoted 6 lines

AFAIU, for the purpose of ttm use, fallback to vmalloc is fine.

* Please note that any use of gfp flags outside of GFP_KERNEL is
careful to not
* fall back to vmalloc.
*

That's weird, because kvcalloc does the same thing. If that were not able to fall back to vmalloc, it would be pretty useless.

    static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags)
    {
             return kvmalloc_array(n, size, flags | __GFP_ZERO);
    }

Maybe kvcalloc is the function we TTM should be using here anyway, instead of open-coding the kvmalloc_array call with an extra GFP flag.

Regards,
Felix

Yes, I agree with your point, and in amdkfd driver code, we have the same risk in svm_range_dma_map_dev().

Best Regards,
Kevin

Quoted 296 lines

Actually the current implementation documents the behavior, but it is deep inside the implementation to be noticeable - at least not obvious while using kvmalloc_array.

Thanks,
Lijo

Quoted 288 lines

Regards,
Christian.

Quoted 284 lines

the system availed memory about 2T, but it will still fail.

2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.
the 5.18 kernel will add this patch to fix this issue .

Best Regards,
Kevin
------------------------------------------------------------------------

*From:* Koenig, Christian <[email protected]> *Sent:* Wednesday, April 20, 2022 8:42 PM
*To:* Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
*Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

Quoted 251 lines

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM) bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

initial_domain = (u32)(0xffffffff & args->in.domains); retry:

        r = amdgpu_gem_object_create(adev, size, args->in.alignment,
                   initial_domain,
                   flags, ttm_bo_type_device, resv, &gobj);
        if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
      flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
      goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
      initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
      goto retry;
                }
DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
              size, initial_domain, args->in.alignment, r);
        }

Best Regards,
Kevin

------------------------------------------------------------------------

*From:* Christian König <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 7:55 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; Koenig, Christian <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

Quoted 176 lines

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree
<https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v5.18-rc3&id=a421ef303008b0ceee2cfc625c3246fa7654b0ca
mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

------------------------------------------------------------------------

*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 6:53 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>
*Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 31 lines

[AMD Official Use Only]

------------------------------------------------------------------------

*From:* Koenig, Christian <[email protected]> <mailto:[email protected]>
*Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>; [email protected] <mailto:[email protected]> <[email protected]> <mailto:[email protected]>
*Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use

vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to vmalloc() when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Quoted 6 lines

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!
*
*The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 79 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...

      if ((flags & GFP_KERNEL) != GFP_KERNEL)
              return kmalloc_node(size, flags, node);

...
}

Best Regards,
Kevin

Quoted 1 line

to allocate memory, when request size is exceeds kmalloc limit,

it will

Quoted 5 lines

cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

<mailto:[email protected]>

Review

Quoted 5 lines
---
  drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---
  1 file changed, 11 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/drivers/gpu/drm/ttm/ttm_tt.c

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object

Quoted 4 lines

index 79c870a3bef8..9f2f3e576b8d 100644

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object

*bo, bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
  static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
  {
       ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
-                     GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;
+
+     memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
       return 0;
  }
 
@@ -108,10 +111,12 @@ static int

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
       ttm->pages = kvmalloc_array(ttm->num_pages,

sizeof(*ttm->pages) +
sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);

+ GFP_KERNEL);
       if (!ttm->pages)
               return -ENOMEM;

+ memset(ttm->pages, 0, ttm->num_pages *

(sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
       ttm->dma_address = (void *)(ttm->pages + ttm->num_pages);
       return 0;
  }
@@ -120,9 +125,12 @@ static int

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int · line 132

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
  {
       ttm->dma_address = kvmalloc_array(ttm->num_pages,
 sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
       if (!ttm->dma_address)
               return -ENOMEM;
+
+     memset(ttm->dma_address, 0, ttm->num_pages *

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
       return 0;
  }
 
#14 lore raw

Am 21.04.22 um 04:15 schrieb Wang, Yang(Kevin):

Quoted 50 lines

[AMD Official Use Only]

------------------------------------------------------------------------
*From:* Kuehling, Felix <[email protected]>
*Sent:* Thursday, April 21, 2022 5:21 AM
*To:* Lazar, Lijo <[email protected]>; Koenig, Christian <[email protected]>; Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]> *Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit

Quoted 22 lines · Lijo
Quoted 12 lines · Christian König

Am 20.04.22 um 14:54 schrieb Wang, Yang(Kevin):

Quoted 10 lines

[AMD Official Use Only]

Hi Chris,

1) Change the test case to use something larger than 1TiB. sure, we can increase the size of BO and make test pass, but if user really want to allocate 1TB GTT BO, we have no reason to let it fail? right?

Quoted 6 lines

No, the reason is the underlying core kernel doesn't allow kvmalloc allocations with GFP_ZERO which are large enough to hold the array of allocated pages for this.

We are working on top of the core Linux kernel and should *NEVER* ever add workarounds like what was suggested here. >

Quoted 6 lines

AFAIU, for the purpose of ttm use, fallback to vmalloc is fine.

 * Please note that any use of gfp flags outside of GFP_KERNEL is careful to not
 * fall back to vmalloc.
 *

That's weird, because kvcalloc does the same thing. If that were not able to fall back to vmalloc, it would be pretty useless.

    static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags) {
             return kvmalloc_array(n, size, flags | __GFP_ZERO);
    }

Quoted 7 lines

Maybe kvcalloc is the function we TTM should be using here anyway, instead of open-coding the kvmalloc_array call with an extra GFP flag.

Regards,
   Felix

Yes, I agree with your point, and in amdkfd driver code, we have the same risk in svm_range_dma_map_dev().

Yes, sounds like a good idea to me as well to change that.

Regards,
Christian.

Quoted 323 lines

Best Regards,
Kevin

Quoted 20 lines

Actually the current implementation documents the behavior, but it is deep inside the implementation to be noticeable - at least not obvious while using kvmalloc_array.

Thanks,
Lijo

Quoted 12 lines

Regards,
Christian.

Quoted 9 lines

the system availed memory about 2T, but it will still fail.

2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.
    the 5.18 kernel will add this patch to fix this issue .

Best Regards,
Kevin

------------------------------------------------------------------------

Quoted 56 lines
Quoted 56 lines
Quoted 56 lines

*From:* Koenig, Christian <[email protected]> *Sent:* Wednesday, April 20, 2022 8:42 PM
*To:* Wang, Yang(Kevin) <[email protected]>; Christian König <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
*Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

yes and that is perfectly valid and expected behavior. There is absolutely no need to change anything in TTM here.

What we could do is:
1) Change the test case to use something larger than 1TiB.
2) Change kvmalloc to allow GFP_ZERO allocations even in the vmalloc fallback path.

Regards,
Christian.

Am 20.04.22 um 14:39 schrieb Wang, Yang(Kevin):

Quoted 33 lines

[AMD Official Use Only]

Hi Chirs,

yes, right, the amdgpu drive rwill use amdgpu_bo_validate_size() function to verify bo size,
but when driver try to allocate VRAM domain bo fail, the amdgpu driver will fall back to allocate domain = (GTT | VRAM)  bo. please check following code, it will cause the 2nd time to allocate bo fail during allocate 256Mb buffer to store dma address (via kvmalloc()).

initial_domain = (u32)(0xffffffff & args->in.domains); retry:
        r = amdgpu_gem_object_create(adev, size, args->in.alignment, initial_domain,
                   flags, ttm_bo_type_device, resv, &gobj); if (r && r != -ERESTARTSYS) {
                if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; goto retry;
                }

                if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
      goto retry;
                }
DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", size, initial_domain, args->in.alignment, r); }

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

Best Regards,
Kevin

------------------------------------------------------------------------

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

*From:* Christian König <[email protected]> <mailto:[email protected]

<mailto:[email protected]>>

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

*Sent:* Wednesday, April 20, 2022 7:55 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected] <mailto:[email protected]>>;

Koenig, Christian

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

<[email protected]> <mailto:[email protected]

<mailto:[email protected]>>;

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

[email protected]
<mailto:[email protected]

<mailto:[email protected]>>

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

<[email protected]>
<mailto:[email protected]

<mailto:[email protected]>>;

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

[email protected]
<mailto:[email protected]

<mailto:[email protected]>>

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

<[email protected]>

<mailto:[email protected] <mailto:[email protected]>>

Quoted 43 lines
Quoted 43 lines
Quoted 43 lines
Quoted 43 lines

*Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Hi Kevin,

no, the test case should already fail in amdgpu_bo_validate_size().

If we have a system with 2TiB of memory where the test case could succeed then we should increase the requested size to something larger.

And if the underlying core Linux kernel functions don't allow allocations as large as the requested one we should *NEVER* ever add workarounds like this.

It is perfectly expected that this test case is not able to fulfill the desired allocation. That it fails during kvmalloc is unfortunate, but not a show stopper.

Regards,
Christian.

Am 20.04.22 um 13:32 schrieb Wang, Yang(Kevin):

Quoted 20 lines

[AMD Official Use Only]

Hi Chris,

you misunderstood background about this case.

although we expect this test case to fail, it should fail at the location where the Bo actual memory is actually allocated. now the code logic will cause the failure to allocate memory to store DMA address.

e.g: the case is failed in 2TB system ram machine, it should be allocated successful, but it is failed.

allocate 1TB BO, the ttm should allocate 1TB/4k * 8 buffer to store allocate result (page address), this should not be failed usually.

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

There is a similar fix in upstream kernel 5.18, before this fix entered the TTM code, this problem existed in TTM.

kernel/git/torvalds/linux.git - Linux kernel source tree

<https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=v5.18-rc3&id=a421ef303008b0ceee2cfc625c3246fa7654b0ca

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

mm: allow !GFP_KERNEL allocations for kvmalloc

Best Regards,
Kevin

------------------------------------------------------------------------

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*From:* Koenig, Christian <[email protected]> <mailto:[email protected] <mailto:[email protected]>> *Sent:* Wednesday, April 20, 2022 6:53 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected] <mailto:[email protected]>>;

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

<mailto:[email protected]

<mailto:[email protected]>>

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

<[email protected]>
<mailto:[email protected]

<mailto:[email protected]>>;

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

[email protected]
<mailto:[email protected]

<mailto:[email protected]>>

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

<[email protected]>
<mailto:[email protected]

<mailto:[email protected]>>

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

*Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 11:07 schrieb Wang, Yang(Kevin):

Quoted 6 lines

[AMD Official Use Only]

------------------------------------------------------------------------

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*From:* Koenig, Christian <[email protected]> <mailto:[email protected] <mailto:[email protected]>> *Sent:* Wednesday, April 20, 2022 5:00 PM
*To:* Wang, Yang(Kevin) <[email protected]> <mailto:[email protected]

<mailto:[email protected]>>; [email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

<mailto:[email protected]

<mailto:[email protected]>>

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

<[email protected]>
<mailto:[email protected]

<mailto:[email protected]>>;

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

[email protected]
<mailto:[email protected]

<mailto:[email protected]>>

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

<[email protected]>
<mailto:[email protected]

<mailto:[email protected]>>

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

*Subject:* Re: [PATCH] drm/ttm: fix ttm tt init fail when size exceeds kmalloc limit
Am 20.04.22 um 10:56 schrieb Yang Wang:

Quoted 1 line

if the __GFP_ZERO is set, the kvmalloc() can't fallback to use

vmalloc()

Hui what? Why should kvmalloc() not be able to fallback to

vmalloc()

Quoted 106 lines
Quoted 106 lines
Quoted 105 lines
Quoted 104 lines
Quoted 103 lines
Quoted 11 lines

when __GFP_ZERO is set?

And even that is really the case then that sounds like a bug in kvmalloc().

Regards,
Christian.

[kevin]:
it is really test case from libdrm amdgpu test, which try to allocate a big BO which will cause ttm tt init fail.

LOL! Guys, this test case is intended to fail!
*
*The test consists of allocating a buffer so ridiculous large that it should never succeed and be rejected by the kernel driver.

This patch here is a really clear NAK.

Regards,
Christian.

Quoted 79 lines

it may be a kvmalloc() bug, and this patch can as a workaround in ttm before this issue fix.

void *kvmalloc_node(size_t size, gfp_t flags, int node) {
...
      if ((flags & GFP_KERNEL) != GFP_KERNEL) return kmalloc_node(size, flags, node);
...
}

Best Regards,
Kevin

Quoted 1 line

to allocate memory, when request size is exceeds kmalloc limit,

it will

Quoted 5 lines

cause allocate memory fail.

e.g: when ttm want to create a BO with 1TB size, it maybe fail.

<mailto:[email protected] <mailto:[email protected]>>

Review

Quoted 5 lines

---
drivers/gpu/drm/ttm/ttm_tt.c | 14 +++++++++++---

   1 file changed, 11 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/drivers/gpu/drm/ttm/ttm_tt.c

drivers/gpu/drm/ttm/ttm_tt.c int ttm_tt_create(struct ttm_buffer_object

Quoted 4 lines

index 79c870a3bef8..9f2f3e576b8d 100644

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -97,9 +97,12 @@ int ttm_tt_create(struct ttm_buffer_object

*bo, bool zero_alloc)

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 14 lines
drivers/gpu/drm/ttm/ttm_tt.c +4 -1
    static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
    {
         ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->pages)
                 return -ENOMEM;
+
+ memset(ttm->pages, 0, ttm->num_pages * sizeof(void *));
+
         return 0;
    }
 
@@ -108,10 +111,12 @@ static int

ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm)

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +0 -0
         ttm->pages = kvmalloc_array(ttm->num_pages,

sizeof(*ttm->pages) +
sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
        if (!ttm->pages)
                return -ENOMEM;

+ memset(ttm->pages, 0, ttm->num_pages *

(sizeof(*ttm->pages) + sizeof(*ttm->dma_address)));

drivers/gpu/drm/ttm/ttm_tt.c static int

Quoted 5 lines

+
        ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); return 0;
   }

drivers/gpu/drm/ttm/ttm_tt.c +0 -0
@@ -120,9 +125,12 @@ static int

ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm)

drivers/gpu/drm/ttm/ttm_tt.c static int · line 132

Quoted 9 lines
drivers/gpu/drm/ttm/ttm_tt.c +3 -1
    {
         ttm->dma_address = kvmalloc_array(ttm->num_pages,
 sizeof(*ttm->dma_address),
- GFP_KERNEL | __GFP_ZERO);
+ GFP_KERNEL);
         if (!ttm->dma_address)
                 return -ENOMEM;
+
+ memset(ttm->dma_address, 0, ttm->num_pages *

sizeof(*ttm->dma_address));

Quoted 4 lines
drivers/gpu/drm/ttm/ttm_tt.c +1 -0
+
         return 0;
    }