Skip to contentSkip to conversation

x86: Fix ptrdiff_t for x86_64

PATCH In review Richard Weinberger 3 participants 10 messages Aug 2, 2024 – Aug 16, 2024 x86

lore
Share & tools
lore
#0 lore raw

PATCH 1/4 x86: Fix ptrdiff_t for x86_64

sbrk() assumes ptrdiff_t is large enough to enlarge/shrink the heap by LONG_MIN/LONG_MAX.
So, use the long type, also to match the rest of the Linux ecosystem.

---
 arch/x86/include/asm/posix_types.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/include/asm/posix_types.h +2 -1
index dbcea7f47f..e1ed9bcabc 100644
@@ -20,11 +20,12 @@ typedef unsigned short	__kernel_gid_t;
 #if defined(__x86_64__)
 typedef unsigned long	__kernel_size_t;
 typedef long		__kernel_ssize_t;
+typedef long		__kernel_ptrdiff_t;
 #else
 typedef unsigned int	__kernel_size_t;
 typedef int		__kernel_ssize_t;
-#endif
 typedef int		__kernel_ptrdiff_t;
+#endif
 typedef long		__kernel_time_t;
 typedef long		__kernel_suseconds_t;
 typedef long		__kernel_clock_t;
Signature · 1 line
2.35.3
#1 lore raw

PATCH 2/4 dlmalloc: Fix integer overflow in request2size()

req is of type size_t, casting it to long opens the door
for an integer overflow.
Values between LONG_MAX - (SIZE_SZ + MALLOC_ALIGN_MASK) - 1 and LONG_MAX cause and overflow such that request2size() returns MINSIZE.

Fix by removing the cast.
The origin of the cast is unclear, it's in u-boot and ppcboot since ever and predates the CVS history.
Doug Lea's original dlmalloc implementation also doesn't have it.

---
 common/dlmalloc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
common/dlmalloc.c +2 -2
index 62e8557daa..44b06e38b2 100644
@@ -386,8 +386,8 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 /* pad request bytes into a usable size */
 
 #define request2size(req) \
- (((long)((req) + (SIZE_SZ + MALLOC_ALIGN_MASK)) < \
-  (long)(MINSIZE + MALLOC_ALIGN_MASK)) ? MINSIZE : \
+ ((((req) + (SIZE_SZ + MALLOC_ALIGN_MASK)) < \
+  (MINSIZE + MALLOC_ALIGN_MASK)) ? MINSIZE : \
    (((req) + (SIZE_SZ + MALLOC_ALIGN_MASK)) & ~(MALLOC_ALIGN_MASK)))
 
 /* Check if m has acceptable alignment */
Signature · 1 line
2.35.3
Reviewed-by on v1 2/4
#2 lore raw

Re: [PATCH 2/4] dlmalloc: Fix integer overflow in request2size()

Quoted 11 lines · Richard Weinberger

req is of type size_t, casting it to long opens the door
for an integer overflow.
Values between LONG_MAX - (SIZE_SZ + MALLOC_ALIGN_MASK) - 1 and LONG_MAX cause and overflow such that request2size() returns MINSIZE.

Fix by removing the cast.
The origin of the cast is unclear, it's in u-boot and ppcboot since ever and predates the CVS history.
Doug Lea's original dlmalloc implementation also doesn't have it.

Quoted 5 lines
---
 common/dlmalloc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
Quoted 18 lines
common/dlmalloc.c +2 -3
index 62e8557daa..44b06e38b2 100644
@@ -386,8 +386,8 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 /* pad request bytes into a usable size */
 
 #define request2size(req) \
- (((long)((req) + (SIZE_SZ + MALLOC_ALIGN_MASK)) < \
-  (long)(MINSIZE + MALLOC_ALIGN_MASK)) ? MINSIZE : \
+ ((((req) + (SIZE_SZ + MALLOC_ALIGN_MASK)) < \
+  (MINSIZE + MALLOC_ALIGN_MASK)) ? MINSIZE : \
    (((req) + (SIZE_SZ + MALLOC_ALIGN_MASK)) & ~(MALLOC_ALIGN_MASK)))
 
 /* Check if m has acceptable alignment */
--

2.35.3

#3 lore raw

PATCH 3/4 dlmalloc: Fix integer overflow in sbrk()

In reply to Richard Weinberger

Make sure that the new break is within mem_malloc_start and mem_malloc_end before making progress.
ulong new = old + increment; can overflow for extremely large increment values and memset() can get wrongly called.

---
 common/dlmalloc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
common/dlmalloc.c +3 -3
index 44b06e38b2..c8d1da1cb1 100644
@@ -581,6 +581,9 @@ void *sbrk(ptrdiff_t increment)
 	ulong old = mem_malloc_brk;
 	ulong new = old + increment;
 
+	if ((new < mem_malloc_start) || (new > mem_malloc_end))
+		return (void *)MORECORE_FAILURE;
+
 	/*
 	 * if we are giving memory back make sure we clear it out since
 	 * we set MORECORE_CLEARS to 1
@@ -588,9 +591,6 @@ void *sbrk(ptrdiff_t increment)
 	if (increment < 0)
 		memset((void *)new, 0, -increment);
 
-	if ((new < mem_malloc_start) || (new > mem_malloc_end))
-		return (void *)MORECORE_FAILURE;
-
 	mem_malloc_brk = new;
 
 	return (void *)old;
Signature · 1 line
2.35.3
Reviewed-by on v1 3/4
#4 lore raw

Re: [PATCH 3/4] dlmalloc: Fix integer overflow in sbrk()

Quoted 10 lines · Richard Weinberger

Make sure that the new break is within mem_malloc_start and mem_malloc_end before making progress.
ulong new = old + increment; can overflow for extremely large increment values and memset() can get wrongly called.

---
 common/dlmalloc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

Should we update dlmalloc to the new version?

Quoted 28 lines
common/dlmalloc.c +3 -4
index 44b06e38b2..c8d1da1cb1 100644
@@ -581,6 +581,9 @@ void *sbrk(ptrdiff_t increment)
        ulong old = mem_malloc_brk;
        ulong new = old + increment;
 
+       if ((new < mem_malloc_start) || (new > mem_malloc_end))
+               return (void *)MORECORE_FAILURE;
+
        /*
         * if we are giving memory back make sure we clear it out since
         * we set MORECORE_CLEARS to 1
@@ -588,9 +591,6 @@ void *sbrk(ptrdiff_t increment)
        if (increment < 0)
                memset((void *)new, 0, -increment);
 
-       if ((new < mem_malloc_start) || (new > mem_malloc_end))
-               return (void *)MORECORE_FAILURE;
-
        mem_malloc_brk = new;
 
        return (void *)old;
--

2.35.3

#5 lore raw

Re: [PATCH 3/4] dlmalloc: Fix integer overflow in sbrk()

Quoted 7 lines · Simon Glass
Quoted 6 lines · Richard Weinberger

Make sure that the new break is within mem_malloc_start and mem_malloc_end before making progress.
ulong new = old + increment; can overflow for extremely large increment values and memset() can get wrongly called.

Quoted 8 lines
Quoted 4 lines
---
 common/dlmalloc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

Should we update dlmalloc to the new version?

A worthy but non-trivial goal, I think.

Signature · 1 line
Tom
#6 lore raw

PATCH 4/4 dlmalloc: Make sure allocation size is within malloc area

In reply to Richard Weinberger

Since U-Boot does not support memory overcommit we can enforce that the allocation size is within the malloc area. This is a simple and efficient hardening measure to mitigate further integer overflows in dlmalloc.

---
 common/dlmalloc.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
common/dlmalloc.c +6 -3
index c8d1da1cb1..d264fc031a 100644
@@ -1274,7 +1274,8 @@ Void_t* mALLOc_impl(bytes) size_t bytes;
     return NULL;
   }
 
-  if ((long)bytes < 0) return NULL;
+  if (bytes > CONFIG_SYS_MALLOC_LEN || (long)bytes < 0)
+     return NULL;
 
   nb = request2size(bytes);  /* padded request size; */
 
@@ -1687,7 +1688,8 @@ Void_t* rEALLOc_impl(oldmem, bytes) Void_t* oldmem; size_t bytes;
   }
 #endif
 
-  if ((long)bytes < 0) return NULL;
+  if (bytes > CONFIG_SYS_MALLOC_LEN || (long)bytes < 0)
+     return NULL;
 
   /* realloc of null is supposed to be same as malloc */
   if (oldmem == NULL) return mALLOc_impl(bytes);
@@ -1907,7 +1909,8 @@ Void_t* mEMALIGn_impl(alignment, bytes) size_t alignment; size_t bytes;
   mchunkptr remainder;        /* spare room at end to split off */
   long      remainder_size;   /* its size */
 
-  if ((long)bytes < 0) return NULL;
+  if (bytes > CONFIG_SYS_MALLOC_LEN || (long)bytes < 0)
+     return NULL;
 
 #if CONFIG_IS_ENABLED(SYS_MALLOC_F)
 	if (!(gd->flags & GD_FLG_FULL_MALLOC_INIT)) {
Signature · 1 line
2.35.3
Reviewed-by on v1 4/4
#7 lore raw

Re: [PATCH 4/4] dlmalloc: Make sure allocation size is within malloc area

Hi Richard,

Quoted 46 lines · Richard Weinberger

Since U-Boot does not support memory overcommit we can enforce that the allocation size is within the malloc area. This is a simple and efficient hardening measure to mitigate further integer overflows in dlmalloc.

---
 common/dlmalloc.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
common/dlmalloc.c +6 -4
index c8d1da1cb1..d264fc031a 100644
@@ -1274,7 +1274,8 @@ Void_t* mALLOc_impl(bytes) size_t bytes;
     return NULL;
   }
 
-  if ((long)bytes < 0) return NULL;
+  if (bytes > CONFIG_SYS_MALLOC_LEN || (long)bytes < 0)
+     return NULL;
 
   nb = request2size(bytes);  /* padded request size; */
 
@@ -1687,7 +1688,8 @@ Void_t* rEALLOc_impl(oldmem, bytes) Void_t* oldmem; size_t bytes;
   }
 #endif
 
-  if ((long)bytes < 0) return NULL;
+  if (bytes > CONFIG_SYS_MALLOC_LEN || (long)bytes < 0)
+     return NULL;
 
   /* realloc of null is supposed to be same as malloc */
   if (oldmem == NULL) return mALLOc_impl(bytes);
@@ -1907,7 +1909,8 @@ Void_t* mEMALIGn_impl(alignment, bytes) size_t alignment; size_t bytes;
   mchunkptr remainder;        /* spare room at end to split off */
   long      remainder_size;   /* its size */
 
-  if ((long)bytes < 0) return NULL;
+  if (bytes > CONFIG_SYS_MALLOC_LEN || (long)bytes < 0)
+     return NULL;
 
 #if CONFIG_IS_ENABLED(SYS_MALLOC_F)
        if (!(gd->flags & GD_FLG_FULL_MALLOC_INIT)) {
--
Quoted 2 lines

2.35.3

I wonder if we can get away without the memalign() one since it is calling malloc() always? There is still the request2size() though.

Regards,
Simon

Reviewed-by on v1 1/4
#8 lore raw
In reply to Richard Weinberger
Quoted 10 lines · Richard Weinberger

sbrk() assumes ptrdiff_t is large enough to enlarge/shrink the heap by LONG_MIN/LONG_MAX.
So, use the long type, also to match the rest of the Linux ecosystem.

---
 arch/x86/include/asm/posix_types.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
Quoted 21 lines
arch/x86/include/asm/posix_types.h +2 -2
index dbcea7f47f..e1ed9bcabc 100644
@@ -20,11 +20,12 @@ typedef unsigned short      __kernel_gid_t;
 #if defined(__x86_64__)
 typedef unsigned long  __kernel_size_t;
 typedef long           __kernel_ssize_t;
+typedef long           __kernel_ptrdiff_t;
 #else
 typedef unsigned int   __kernel_size_t;
 typedef int            __kernel_ssize_t;
-#endif
 typedef int            __kernel_ptrdiff_t;
+#endif
 typedef long           __kernel_time_t;
 typedef long           __kernel_suseconds_t;
 typedef long           __kernel_clock_t;
--

2.35.3

#9 lore raw
In reply to Richard Weinberger
Quoted 5 lines · Richard Weinberger

sbrk() assumes ptrdiff_t is large enough to enlarge/shrink the heap by LONG_MIN/LONG_MAX.
So, use the long type, also to match the rest of the Linux ecosystem.

Applied to u-boot/next, thanks!

Signature · 1 line
Tom