Skip to contentSkip to conversation

perf: Fix vmalloc ring buffer free function

PATCH Superseded Jiri Olsa 5 participants 28 messages Mar 1, 2013 – May 2, 2013 perf events

lore
Share & tools
lore

Jiri Olsa

author
#0 lore raw

PATCH v2 1/1 perf: Fix vmalloc ring buffer free function

If we allocate perf ring buffer with the size of single page, we will get memory corruption when releasing it. It's caused by rb_free_work function (CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Introducing page_nr function (CONFIG_PERF_USE_VMALLOC only) that returns number of allocated pages. Using it in rb_free_work and perf_mmap_to_page functions.

Also setting rb->nr_pages to 0 in case we have only user page allocated, which will fail perf_output_begin function and prevents sample storage.

v2 changes:
- fixed the perf_output_begin handling of single page buffer

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
---
 kernel/events/ring_buffer.c | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)
kernel/events/ring_buffer.c +33 -7
index 23cb34f..a802151 100644
@@ -154,7 +154,8 @@ int perf_output_begin(struct perf_output_handle *handle,
 	if (head - local_read(&rb->wakeup) > rb->watermark)
 		local_add(rb->watermark, &rb->wakeup);
 
-	handle->page = offset >> (PAGE_SHIFT + page_order(rb));
+	/* page is allways 0 for CONFIG_PERF_USE_VMALLOC option */
+	handle->page = offset >> PAGE_SHIFT;
 	handle->page &= rb->nr_pages - 1;
 	handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1);
 	handle->addr = rb->data_pages[handle->page];
@@ -312,11 +313,21 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+	return page_order(rb) == -1 ?
+		1 :                        /* no data, just user page */
+		1 + (1 << page_order(rb)); /* user page + data pages */
+}
 
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > page_nr(rb))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +347,10 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = page_nr(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	for (i = 0; i < nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -371,9 +382,24 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 		goto fail_all_buf;
 
 	rb->user_page = all_buf;
-	rb->data_pages[0] = all_buf + PAGE_SIZE;
-	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+
+	/*
+	 * For special case nr_pages == 0 we have
+	 * only the user page mmaped plus:
+	 *
+	 *   rb->data_pages[0] = NULL
+	 *   rb->nr_pages      = 0
+	 *   rb->page_order    = -1
+	 *
+	 * The perf_output_begin function is guarded
+	 * by (rb->nr_pages > 0) condition, so no
+	 * output code touches above setup if we
+	 * have only user page allocated.
+	 */
+
+	rb->data_pages[0] = nr_pages ? all_buf + PAGE_SIZE : NULL;
+	rb->nr_pages      = nr_pages ? 1 : 0;
+	rb->page_order    = ilog2(nr_pages);
 
 	ring_buffer_init(rb, watermark, flags);
 
Signature · 1 line
1.7.11.7
#1 lore raw
perf: Fix vmalloc ring buffer free function
Commit-ID:  12bc915ee149ac31d17c513edc7303660d024239
Gitweb:     http://git.kernel.org/tip/12bc915ee149ac31d17c513edc7303660d024239
Author:     Jiri Olsa <[email protected]>
AuthorDate: Fri, 1 Mar 2013 17:34:49 +0100
Committer:  Ingo Molnar <[email protected]>
CommitDate: Wed, 6 Mar 2013 11:14:01 +0100

perf: Fix vmalloc ring buffer free function

If we allocate perf ring buffer with the size of single page, we will get memory corruption when releasing it. It's caused by rb_free_work function (CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Introducing page_nr function (CONFIG_PERF_USE_VMALLOC only) that returns number of allocated pages. Using it in rb_free_work and perf_mmap_to_page functions.

Also setting rb->nr_pages to 0 in case we have only user page allocated, which will fail perf_output_begin function and prevents sample storage.

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
Cc "David S. Miller" <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Andrew Morton <[email protected]>
---
 kernel/events/ring_buffer.c | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)
kernel/events/ring_buffer.c +33 -7
index 23cb34f..a802151 100644
@@ -154,7 +154,8 @@ int perf_output_begin(struct perf_output_handle *handle,
 	if (head - local_read(&rb->wakeup) > rb->watermark)
 		local_add(rb->watermark, &rb->wakeup);
 
-	handle->page = offset >> (PAGE_SHIFT + page_order(rb));
+	/* page is allways 0 for CONFIG_PERF_USE_VMALLOC option */
+	handle->page = offset >> PAGE_SHIFT;
 	handle->page &= rb->nr_pages - 1;
 	handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1);
 	handle->addr = rb->data_pages[handle->page];
@@ -312,11 +313,21 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+	return page_order(rb) == -1 ?
+		1 :                        /* no data, just user page */
+		1 + (1 << page_order(rb)); /* user page + data pages */
+}
 
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > page_nr(rb))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +347,10 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = page_nr(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	for (i = 0; i < nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -371,9 +382,24 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 		goto fail_all_buf;
 
 	rb->user_page = all_buf;
-	rb->data_pages[0] = all_buf + PAGE_SIZE;
-	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+
+	/*
+	 * For special case nr_pages == 0 we have
+	 * only the user page mmaped plus:
+	 *
+	 *   rb->data_pages[0] = NULL
+	 *   rb->nr_pages      = 0
+	 *   rb->page_order    = -1
+	 *
+	 * The perf_output_begin function is guarded
+	 * by (rb->nr_pages > 0) condition, so no
+	 * output code touches above setup if we
+	 * have only user page allocated.
+	 */
+
+	rb->data_pages[0] = nr_pages ? all_buf + PAGE_SIZE : NULL;
+	rb->nr_pages      = nr_pages ? 1 : 0;
+	rb->page_order    = ilog2(nr_pages);
 
 	ring_buffer_init(rb, watermark, flags);
 
#2 lore raw
In reply to Jiri Olsa

2013/3/1 Jiri Olsa <[email protected]>:

Quoted 50 lines

If we allocate perf ring buffer with the size of single page, we will get memory corruption when releasing it. It's caused by rb_free_work function (CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Introducing page_nr function (CONFIG_PERF_USE_VMALLOC only) that returns number of allocated pages. Using it in rb_free_work and perf_mmap_to_page functions.

Also setting rb->nr_pages to 0 in case we have only user page allocated, which will fail perf_output_begin function and prevents sample storage.

v2 changes:
- fixed the perf_output_begin handling of single page buffer

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
---
 kernel/events/ring_buffer.c | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)
kernel/events/ring_buffer.c +2 -1
index 23cb34f..a802151 100644
@@ -154,7 +154,8 @@ int perf_output_begin(struct perf_output_handle *handle,
        if (head - local_read(&rb->wakeup) > rb->watermark)
                local_add(rb->watermark, &rb->wakeup);
 
-       handle->page = offset >> (PAGE_SHIFT + page_order(rb));
+       /* page is allways 0 for CONFIG_PERF_USE_VMALLOC option */
+       handle->page = offset >> PAGE_SHIFT;
        handle->page &= rb->nr_pages - 1;
        handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1);
        handle->addr = rb->data_pages[handle->page];
@@ -312,11 +313,21 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 325

Quoted 10 lines
kernel/events/ring_buffer.c +10 -0
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+       return page_order(rb) == -1 ?
+               1 :                        /* no data, just user page */
+               1 + (1 << page_order(rb)); /* user page + data pages */
+}

Could be simply rb->nr_page + 1 ?

Patch looks good in any case. Thanks.

author
#3 lore raw
Quoted 64 lines · Frederic Weisbecker

2013/3/1 Jiri Olsa <[email protected]>:

Quoted 50 lines

If we allocate perf ring buffer with the size of single page, we will get memory corruption when releasing it. It's caused by rb_free_work function (CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Introducing page_nr function (CONFIG_PERF_USE_VMALLOC only) that returns number of allocated pages. Using it in rb_free_work and perf_mmap_to_page functions.

Also setting rb->nr_pages to 0 in case we have only user page allocated, which will fail perf_output_begin function and prevents sample storage.

v2 changes:
- fixed the perf_output_begin handling of single page buffer

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
---
 kernel/events/ring_buffer.c | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)
kernel/events/ring_buffer.c +2 -1
index 23cb34f..a802151 100644
@@ -154,7 +154,8 @@ int perf_output_begin(struct perf_output_handle *handle,
        if (head - local_read(&rb->wakeup) > rb->watermark)
                local_add(rb->watermark, &rb->wakeup);
 
-       handle->page = offset >> (PAGE_SHIFT + page_order(rb));
+       /* page is allways 0 for CONFIG_PERF_USE_VMALLOC option */
+       handle->page = offset >> PAGE_SHIFT;
        handle->page &= rb->nr_pages - 1;
        handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1);
        handle->addr = rb->data_pages[handle->page];
@@ -312,11 +313,21 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 325

Quoted 10 lines
kernel/events/ring_buffer.c +10 -0
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+       return page_order(rb) == -1 ?
+               1 :                        /* no data, just user page */
+               1 + (1 << page_order(rb)); /* user page + data pages */
+}

Could be simply rb->nr_page + 1 ?

Quoted 1 line

Patch looks good in any case. Thanks.

nope, because CONFIG_PERF_USE_VMALLOC rb uses only 1st slot of rg->data_pages[], so rb->nr_page is either 0 or 1

the actuall number of pages is counted via rb->page_order (which is -1 for case with no data pages)

thanks,
jirka

#4 lore raw

2013/3/6 Jiri Olsa <[email protected]>:

Quoted 5 lines

nope, because CONFIG_PERF_USE_VMALLOC rb uses only 1st slot of rg->data_pages[], so rb->nr_page is either 0 or 1

the actuall number of pages is counted via rb->page_order (which is -1 for case with no data pages)

Ah right.

#5 lore raw
In reply to Jiri Olsa
Quoted 37 lines · Jiri Olsa

If we allocate perf ring buffer with the size of single page, we will get memory corruption when releasing it. It's caused by rb_free_work function (CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Introducing page_nr function (CONFIG_PERF_USE_VMALLOC only) that returns number of allocated pages. Using it in rb_free_work and perf_mmap_to_page functions.

Also setting rb->nr_pages to 0 in case we have only user page allocated, which will fail perf_output_begin function and prevents sample storage.

v2 changes:
- fixed the perf_output_begin handling of single page buffer

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
---
 kernel/events/ring_buffer.c | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)
kernel/events/ring_buffer.c +0 -0
index 23cb34f..a802151 100644

kernel/events/ring_buffer.c int perf_output_begin(struct perf_output_handle *handle, · line 158

Quoted 6 lines
kernel/events/ring_buffer.c +2 -1
@@ -154,7 +154,8 @@ int perf_output_begin(struct perf_output_handle *handle,
 	if (head - local_read(&rb->wakeup) > rb->watermark)
 		local_add(rb->watermark, &rb->wakeup);
 
-	handle->page = offset >> (PAGE_SHIFT + page_order(rb));
+	/* page is allways 0 for CONFIG_PERF_USE_VMALLOC option */
+	handle->page = offset >> PAGE_SHIFT;

I don't get that comment.. also it makes the calculation for page inconsistent with the below calculation for addr.

We basically want to split the offset into a page number and an offset within that; this means we need:

pg_nr = offset >> page_shift;
pg_offset = offset & (1 << page_shift) - 1;

You just wrecked that.

Quoted 7 lines
kernel/events/ring_buffer.c +0 -0
 	handle->page &= rb->nr_pages - 1;
 	handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1);
 	handle->addr = rb->data_pages[handle->page];
@@ -312,11 +313,21 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 325

Quoted 10 lines
kernel/events/ring_buffer.c +10 -0
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+	return page_order(rb) == -1 ?
+		1 :                        /* no data, just user page */
+		1 + (1 << page_order(rb)); /* user page + data pages */
+}

I think a number of the bugs below is due to the conflation of data pages vs total pages. It might be best to call this data_page_nr() and leave the +1 for the sites where its needed.

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 331

Quoted 6 lines
kernel/events/ring_buffer.c +1 -1
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > page_nr(rb))
 		return NULL;

This is just wrong.. you have page_nr() be 1+2^n, but the comparison is '>' not '>=', this means we get a range of 2+2^n, not the desired 1+2^n.

Quoted 42 lines
kernel/events/ring_buffer.c +20 -5
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +347,10 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = page_nr(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	for (i = 0; i < nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -371,9 +382,24 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 		goto fail_all_buf;
 
 	rb->user_page = all_buf;
-	rb->data_pages[0] = all_buf + PAGE_SIZE;
-	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+
+	/*
+	 * For special case nr_pages == 0 we have
+	 * only the user page mmaped plus:
+	 *
+	 *   rb->data_pages[0] = NULL
+	 *   rb->nr_pages      = 0
+	 *   rb->page_order    = -1
+	 *
+	 * The perf_output_begin function is guarded
+	 * by (rb->nr_pages > 0) condition, so no
+	 * output code touches above setup if we
+	 * have only user page allocated.
+	 */
+
+	rb->data_pages[0] = nr_pages ? all_buf + PAGE_SIZE : NULL;
+	rb->nr_pages      = nr_pages ? 1 : 0;
+	rb->page_order    = ilog2(nr_pages);
 
 	ring_buffer_init(rb, watermark, flags);
 
author
#6 lore raw

heya ;)

great to hear from you again!

Quoted 2 lines · Peter Zijlstra
Quoted 1 line · Jiri Olsa

If we allocate perf ring buffer with the size of single page,

SNIP

Quoted 15 lines

kernel/events/ring_buffer.c int perf_output_begin(struct perf_output_handle *handle, · line 158

Quoted 11 lines
kernel/events/ring_buffer.c +2 -1
index 23cb34f..a802151 100644
@@ -154,7 +154,8 @@ int perf_output_begin(struct perf_output_handle *handle,
 	if (head - local_read(&rb->wakeup) > rb->watermark)
 		local_add(rb->watermark, &rb->wakeup);
 
-	handle->page = offset >> (PAGE_SHIFT + page_order(rb));
+	/* page is allways 0 for CONFIG_PERF_USE_VMALLOC option */
+	handle->page = offset >> PAGE_SHIFT;

I don't get that comment.. also it makes the calculation for page inconsistent with the below calculation for addr.

Quoted 9 lines

We basically want to split the offset into a page number and an offset within that; this means we need:

pg_nr = offset >> page_shift;
pg_offset = offset & (1 << page_shift) - 1;

You just wrecked that.

Quoted 1 line
kernel/events/ring_buffer.c +0 -0
 	handle->page &= rb->nr_pages - 1;

here's ^^^ where the handle->page becomes 0 due to (rb->nr_pages == 0)

for CONFIG_PERF_USE_VMALLOC we use only the first item in rb->data_pages[] array, which holds the whole data memory, and got accessed by 'offset' directly

Quoted 5 lines
Quoted 5 lines
kernel/events/ring_buffer.c +0 -0
 	handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1);
 	handle->addr = rb->data_pages[handle->page];
@@ -312,11 +313,21 @@ void rb_free(struct ring_buffer *rb)
 }
 
Quoted 15 lines

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 325

Quoted 11 lines
kernel/events/ring_buffer.c +10 -0
 #else
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+	return page_order(rb) == -1 ?
+		1 :                        /* no data, just user page */
+		1 + (1 << page_order(rb)); /* user page + data pages */
+}

I think a number of the bugs below is due to the conflation of data pages vs total pages. It might be best to call this data_page_nr() and leave the +1 for the sites where its needed.

both places using page_nr need total pages count, maybe I can rename it into total_page_nr()

Quoted 11 lines

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 331

Quoted 8 lines
kernel/events/ring_buffer.c +1 -1
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > page_nr(rb))
 		return NULL;

This is just wrong.. you have page_nr() be 1+2^n, but the comparison is

Quoted 1 line

'>' not '>=', this means we get a range of 2+2^n, not the desired 1+2^n.

ouch, missed that one

thanks,
jirka

#7 lore raw
Quoted 9 lines · Jiri Olsa
Quoted 8 lines

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 331

Quoted 6 lines

struct page *
perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff) {

kernel/events/ring_buffer.c +1 -1
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > page_nr(rb))
 		return NULL;

This is just wrong.. you have page_nr() be 1+2^n, but the comparison is

Quoted 3 lines
Quoted 1 line

'>' not '>=', this means we get a range of 2+2^n, not the desired 1+2^n.

ouch, missed that one

So, because this is perf/urgent which I want to send Linuswards, I removed the commit to not hold up other bits - please resend the whole patch once everything is fixed.

Thanks,

Ingo

#8 lore raw
In reply to Jiri Olsa
Quoted 8 lines · Jiri Olsa
Quoted 6 lines

pg_nr = offset >> page_shift;
pg_offset = offset & (1 << page_shift) - 1;

You just wrecked that.

Quoted 1 line
    handle->page &= rb->nr_pages - 1;

here's ^^^ where the handle->page becomes 0 due to (rb->nr_pages == 0)

then that'll be &= -1, which is a nop, no?

Also, you wrecked it for anything that uses some intermediate page order (we currently don't).

author
#9 lore raw
Quoted 11 lines · Peter Zijlstra
Quoted 8 lines · Jiri Olsa
Quoted 6 lines

pg_nr = offset >> page_shift;
pg_offset = offset & (1 << page_shift) - 1;

You just wrecked that.

Quoted 1 line
    handle->page &= rb->nr_pages - 1;

here's ^^^ where the handle->page becomes 0 due to (rb->nr_pages == 0)

then that'll be &= -1, which is a nop, no?

ugh.. should have been: '..due to (rb->nr_pages == 1)' as from:

+ rb->nr_pages = nr_pages ? 1 : 0;

if it's 0, it never make it throught the perf_output_begin

Quoted 3 lines

Also, you wrecked it for anything that uses some intermediate page order (we currently don't).

I'll put that hunk back then:

-       handle->page = offset >> (PAGE_SHIFT + page_order(rb));
+       /* page is allways 0 for CONFIG_PERF_USE_VMALLOC option */
+       handle->page = offset >> PAGE_SHIFT;

I did not know there were other plans for that, seemed useless

jirka

#10 lore raw
Quoted 1 line · Jiri Olsa

I did not know there were other plans for that, seemed useless

I like to keep form, confuses me less :-)

Anyway, why are we getting to that part of perf_output_begin() if we don't have any data pages to begin with? Shouldn't we bail before there someplace? Like at the !rb->nr_pages check?

author
#11 lore raw
Quoted 5 lines · Peter Zijlstra
Quoted 1 line · Jiri Olsa

I did not know there were other plans for that, seemed useless

I like to keep form, confuses me less :-)

ook

Quoted 5 lines

Anyway, why are we getting to that part of perf_output_begin() if we don't have any data pages to begin with? Shouldn't we bail before there someplace? Like at the !rb->nr_pages check?

perf_output_begin actually does the !rb->nr_pages check looks like we could check it earlier in some casese, I'll update my todo list ;-)

jirka

author
#12 lore raw

PATCH v3 1/1 perf: Fix vmalloc ring buffer free function

If we allocate perf ring buffer with the size of single page, we will get memory corruption when releasing it. It's caused by rb_free_work function (CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Introducing page_nr function (CONFIG_PERF_USE_VMALLOC only) that returns number of allocated pages. Using it in rb_free_work and perf_mmap_to_page functions.

Also setting rb->nr_pages to 0 in case we have only user page allocated, which will fail perf_output_begin function and prevents sample storage.

v3 changes:
- fixed perf_mmap_to_page check
- keep page_order in handle->page computation

v2 changes:
- fixed the perf_output_begin handling of single page buffer

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
---
 kernel/events/ring_buffer.c | 37 +++++++++++++++++++++++++++++++------
 1 file changed, 31 insertions(+), 6 deletions(-)
kernel/events/ring_buffer.c +31 -6
index 23cb34f..4219be6 100644
@@ -312,11 +312,21 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+	return page_order(rb) == -1 ?
+		1 :                        /* no data, just user page */
+		1 + (1 << page_order(rb)); /* user page + data pages */
+}
 
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff >= page_nr(rb))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +346,10 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = page_nr(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	for (i = 0; i < nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -371,9 +381,24 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 		goto fail_all_buf;
 
 	rb->user_page = all_buf;
-	rb->data_pages[0] = all_buf + PAGE_SIZE;
-	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+
+	/*
+	 * For special case nr_pages == 0 we have
+	 * only the user page mmaped plus:
+	 *
+	 *   rb->data_pages[0] = NULL
+	 *   rb->nr_pages      = 0
+	 *   rb->page_order    = -1
+	 *
+	 * The perf_output_begin function is guarded
+	 * by (rb->nr_pages > 0) condition, so no
+	 * output code touches above setup if we
+	 * have only user page allocated.
+	 */
+
+	rb->data_pages[0] = nr_pages ? all_buf + PAGE_SIZE : NULL;
+	rb->nr_pages      = nr_pages ? 1 : 0;
+	rb->page_order    = ilog2(nr_pages);
 
 	ring_buffer_init(rb, watermark, flags);
 
Signature · 1 line
1.7.11.7
#13 lore raw
Quoted 15 lines · Jiri Olsa
(quoted diff) +10 -0
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+       return page_order(rb) == -1 ?
+               1 :                        /* no data, just user page */
+               1 + (1 << page_order(rb)); /* user page + data pages */
+}
@@ -371,9 +381,24 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
                goto fail_all_buf;
 
        rb->user_page = all_buf;

Review struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) · line 390

Quoted 10 lines
(quoted diff) +4 -3
-       rb->data_pages[0] = all_buf + PAGE_SIZE;
-       rb->page_order = ilog2(nr_pages);
-       rb->nr_pages = 1;
 
+
+       rb->data_pages[0] = nr_pages ? all_buf + PAGE_SIZE : NULL;
+       rb->nr_pages      = nr_pages ? 1 : 0;
+       rb->page_order    = ilog2(nr_pages);
 
        ring_buffer_init(rb, watermark, flags);

Still somewhat confused by this.. wouldn't something 'simple' like the below suffice?

Note how the !vmalloc branch also sets rb->nr_pages = nr_pages.

---
 kernel/events/ring_buffer.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/events/ring_buffer.c +3 -3
index 23cb34f..4f48d02 100644
@@ -316,7 +316,7 @@ void rb_free(struct ring_buffer *rb)
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > rb->nr_pages)
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,7 +336,7 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = rb->nr_pages;
 
 	base = rb->user_page;
 	for (i = 0; i < nr + 1; i++)
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = nr_pages;
 
 	ring_buffer_init(rb, watermark, flags);
 
author
#14 lore raw
Quoted 25 lines · Peter Zijlstra
Quoted 24 lines · Jiri Olsa
(quoted diff) +14 -3
+/*
+ * Returns the total number of pages allocated
+ * by ring buffer including the user page.
+ */
+static int page_nr(struct ring_buffer *rb)
+{
+       return page_order(rb) == -1 ?
+               1 :                        /* no data, just user page */
+               1 + (1 << page_order(rb)); /* user page + data pages */
+}
@@ -371,9 +381,24 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
                goto fail_all_buf;
 
        rb->user_page = all_buf;
-       rb->data_pages[0] = all_buf + PAGE_SIZE;
-       rb->page_order = ilog2(nr_pages);
-       rb->nr_pages = 1;
 
+
+       rb->data_pages[0] = nr_pages ? all_buf + PAGE_SIZE : NULL;
+       rb->nr_pages      = nr_pages ? 1 : 0;
+       rb->page_order    = ilog2(nr_pages);
 
Quoted 6 lines

kernel/events/ring_buffer.c struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) · line 378

Quoted 1 line
kernel/events/ring_buffer.c +0 -0
        ring_buffer_init(rb, watermark, flags);

Still somewhat confused by this.. wouldn't something 'simple' like the below suffice?

Note how the !vmalloc branch also sets rb->nr_pages = nr_pages.

yep it would ;-) though I find it little confusing for:

Quoted 10 lines
---
 kernel/events/ring_buffer.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/events/ring_buffer.c +0 -0
index 23cb34f..4f48d02 100644

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 320

Quoted 6 lines
kernel/events/ring_buffer.c +1 -1
@@ -316,7 +316,7 @@ void rb_free(struct ring_buffer *rb)
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > rb->nr_pages)
 		return NULL;

here it's the '>' that masks that there's actually (rb->nr_pages + 1) pages

kernel/events/ring_buffer.c static void rb_free_work(struct work_struct *work) · line 342

Quoted 11 lines
kernel/events/ring_buffer.c +1 -1
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,7 +336,7 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = rb->nr_pages;
 
 	base = rb->user_page;
 	for (i = 0; i < nr + 1; i++)

and here's the user plus one

kernel/events/ring_buffer.c struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) · line 378

Quoted 8 lines
kernel/events/ring_buffer.c +1 -1
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = nr_pages;
 
 	ring_buffer_init(rb, watermark, flags);

but I'll get used to it np.. ;)

jirka

#15 lore raw
Quoted 15 lines · Jiri Olsa

kernel/events/ring_buffer.c void rb_free(struct ring_buffer *rb) · line 320

Quoted 7 lines
kernel/events/ring_buffer.c +1 -1
@@ -316,7 +316,7 @@ void rb_free(struct ring_buffer *rb)
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-     if (pgoff > (1UL << page_order(rb)))
+     if (pgoff > rb->nr_pages)
              return NULL;

here it's the '>' that masks that there's actually (rb->nr_pages + 1) pages

Quoted 5 lines
kernel/events/ring_buffer.c +0 -0
      return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,7 +336,7 @@ static void rb_free_work(struct work_struct *work)
      int i, nr;
 
Quoted 8 lines

kernel/events/ring_buffer.c static void rb_free_work(struct work_struct *work) · line 342

Quoted 6 lines
kernel/events/ring_buffer.c +1 -1
      rb = container_of(work, struct ring_buffer, work);
-     nr = 1 << page_order(rb);
+     nr = rb->nr_pages;
 
      base = rb->user_page;
      for (i = 0; i < nr + 1; i++)

and here's the user plus one

Right, we could make that either '< nr' and '<= nr' or '<= nr+1' and '< nr+1'. No real preference either way, but you're right in that being consistent here makes sense.

author
#16 lore raw
In reply to Peter Zijlstra
Quoted 1 line · Peter Zijlstra

On Tue, 2013-03-12 at 11:05 +0100, Jiri Olsa wrote:

SNIP

Quoted 27 lines
---
 kernel/events/ring_buffer.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/events/ring_buffer.c +2 -2
index 23cb34f..4f48d02 100644
@@ -316,7 +316,7 @@ void rb_free(struct ring_buffer *rb)
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > rb->nr_pages)
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,7 +336,7 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = rb->nr_pages;
 
 	base = rb->user_page;
 	for (i = 0; i < nr + 1; i++)

kernel/events/ring_buffer.c struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) · line 377

Quoted 6 lines
kernel/events/ring_buffer.c +1 -1
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = nr_pages;
 

hum, and this ^^^ breaks perf_data_size(rb) ;)

static inline unsigned long perf_data_size(struct ring_buffer *rb)

{
        return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
}

we could make it ifdef-ed for CONFIG_PERF_USE_VMALLOC

jirka

#17 lore raw
Quoted 10 lines · Jiri Olsa

kernel/events/ring_buffer.c struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) · line 377

Quoted 7 lines
kernel/events/ring_buffer.c +1 -1
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
      rb->user_page = all_buf;
      rb->data_pages[0] = all_buf + PAGE_SIZE;
      rb->page_order = ilog2(nr_pages);
-     rb->nr_pages = 1;
+     rb->nr_pages = nr_pages;
 

hum, and this ^^^ breaks perf_data_size(rb) ;)

Quoted 4 lines

static inline unsigned long perf_data_size(struct ring_buffer *rb)

{
        return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
}

How so? 0 << n keeps being 0, right?

author
#18 lore raw
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Jiri Olsa

kernel/events/ring_buffer.c struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) · line 377

Quoted 7 lines
kernel/events/ring_buffer.c +1 -1
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
      rb->user_page = all_buf;
      rb->data_pages[0] = all_buf + PAGE_SIZE;
      rb->page_order = ilog2(nr_pages);
-     rb->nr_pages = 1;
+     rb->nr_pages = nr_pages;
 

hum, and this ^^^ breaks perf_data_size(rb) ;)

Quoted 7 lines
Quoted 4 lines

static inline unsigned long perf_data_size(struct ring_buffer *rb)

{
        return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
}

How so? 0 << n keeps being 0, right?

you do 'rb->nr_pages = nr_pages' which was 1 before.. that will bump up the return value

should do '1 << x' but doing 'nr_pages << x' instead

jirka

#19 lore raw

Right you are..

How about something like the below; using that 0 << -1 is still 0.

---
 kernel/events/ring_buffer.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
kernel/events/ring_buffer.c +4 -4
index 23cb34f..e72ca70 100644
@@ -316,7 +316,7 @@ void rb_free(struct ring_buffer *rb)
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > (rb->nr_pages << page_order(rb)))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +336,10 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = rb->nr_pages << page_order(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	for (i = 0; i <= nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = !!nr_pages;
 
 	ring_buffer_init(rb, watermark, flags);
 
author
#20 lore raw
Quoted 3 lines · Peter Zijlstra

Right you are..

How about something like the below; using that 0 << -1 is still 0.

hum.. we are obviously not on the same page wrt 'confusing' ;-)

looks ok, I'll test it

jirka

Quoted 42 lines
---
 kernel/events/ring_buffer.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
kernel/events/ring_buffer.c +4 -4
index 23cb34f..e72ca70 100644
@@ -316,7 +316,7 @@ void rb_free(struct ring_buffer *rb)
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > (rb->nr_pages << page_order(rb)))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +336,10 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = rb->nr_pages << page_order(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	for (i = 0; i <= nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = !!nr_pages;
 
 	ring_buffer_init(rb, watermark, flags);
 
author Tested-by
#21 lore raw
Quoted 8 lines · Jiri Olsa
Quoted 3 lines · Peter Zijlstra

Right you are..

How about something like the below; using that 0 << -1 is still 0.

hum.. we are obviously not on the same page wrt 'confusing' ;-)

looks ok, I'll test it

hi, it work's fine..

Quoted 33 lines

jirka

Quoted 30 lines
---
 kernel/events/ring_buffer.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
kernel/events/ring_buffer.c +3 -3
index 23cb34f..e72ca70 100644
@@ -316,7 +316,7 @@ void rb_free(struct ring_buffer *rb)
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	if (pgoff > (rb->nr_pages << page_order(rb)))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +336,10 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = rb->nr_pages << page_order(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	for (i = 0; i <= nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
Quoted 4 lines

kernel/events/ring_buffer.c struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) · line 374

Quoted 4 lines
kernel/events/ring_buffer.c +0 -0
 	vfree(base);
@@ -373,7 +373,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;

just a nit pick.. in case of having just user page, data_pages[0] holds wrong pointer, but it 'should' be never touch in this case

author
#22 lore raw
Quoted 6 lines · Jiri Olsa
Quoted 5 lines · Jiri Olsa
Quoted 3 lines · Peter Zijlstra

Right you are..

How about something like the below; using that 0 << -1 is still 0.

Quoted 7 lines
Quoted 3 lines

hum.. we are obviously not on the same page wrt 'confusing' ;-)

looks ok, I'll test it

hi, it work's fine..

are you going to include that, or should I repost it?

thanks,
jirka

#23 lore raw
Quoted 2 lines

are you going to include that, or should I repost it?

Ah, please repost (and prettify) it, I'm still very limited in the amount of work that I can do :/

author
#24 lore raw

PATCH v4 1/1 perf: Fix vmalloc ring buffer pages handling

Quoted 6 lines · Peter Zijlstra
Quoted 2 lines

are you going to include that, or should I repost it?

Ah, please repost (and prettify) it, I'm still very limited in the amount of work that I can do :/

ok, attached

jirka

---
If we allocate perf ring buffer with the size of single (user) page, we will get memory corruption when releasing itin rb_free_work function (for CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Adding data_page_nr function that returns number of allocated data pages. Customizing the rest of the code to use it.

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
---
 kernel/events/ring_buffer.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)
kernel/events/ring_buffer.c +10 -4
index 23cb34f..27a1af4 100644
@@ -312,11 +312,16 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else
+static int data_page_nr(struct ring_buffer *rb)
+{
+	return rb->nr_pages << page_order(rb);
+}
 
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	/* The '>' counts in the user page. */
+	if (pgoff > data_page_nr(rb))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +341,11 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = data_page_nr(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	/* The '<=' counts in the user page. */
+	for (i = 0; i <= nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -373,7 +379,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = !!nr_pages;
 
 	ring_buffer_init(rb, watermark, flags);
 
Signature · 1 line
1.7.11.7
author
#25 lore raw

Re: [PATCHv4] perf: Fix vmalloc ring buffer pages handling

Quoted 9 lines · Jiri Olsa
Quoted 6 lines · Peter Zijlstra
Quoted 2 lines

are you going to include that, or should I repost it?

Ah, please repost (and prettify) it, I'm still very limited in the amount of work that I can do :/

ok, attached

Ingo, Peter,

could this be pulled in?

thanks,
jirka

Quoted 77 lines

jirka

---
If we allocate perf ring buffer with the size of single (user) page, we will get memory corruption when releasing itin rb_free_work function (for CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Adding data_page_nr function that returns number of allocated data pages. Customizing the rest of the code to use it.

Cc Corey Ashford <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Namhyung Kim <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
---
 kernel/events/ring_buffer.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)
kernel/events/ring_buffer.c +10 -4
index 23cb34f..27a1af4 100644
@@ -312,11 +312,16 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else
+static int data_page_nr(struct ring_buffer *rb)
+{
+	return rb->nr_pages << page_order(rb);
+}
 
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	/* The '>' counts in the user page. */
+	if (pgoff > data_page_nr(rb))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -336,10 +341,11 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = data_page_nr(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	/* The '<=' counts in the user page. */
+	for (i = 0; i <= nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -373,7 +379,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = !!nr_pages;
 
 	ring_buffer_init(rb, watermark, flags);
 
Signature · 1 line
1.7.11.7
#26 lore raw

Re: [PATCHv4] perf: Fix vmalloc ring buffer pages handling

Quoted 9 lines · Jiri Olsa
Quoted 8 lines · Jiri Olsa
Quoted 6 lines · Peter Zijlstra
Quoted 2 lines

are you going to include that, or should I repost it?

Ah, please repost (and prettify) it, I'm still very limited in the amount of work that I can do :/

Quoted 5 lines
Quoted 1 line

ok, attached

Ingo, Peter,

could this be pulled in?

Yeah, Peter acked it - I will pick it up.

Thanks,

Ingo

#27 lore raw

[tip:perf/urgent] perf: Fix vmalloc ring buffer pages handling

In reply to Jiri Olsa
perf: Fix vmalloc ring buffer pages handling
Commit-ID:  5919b30933d7c4fac1f214c59f26c5e990044f09
Gitweb:     http://git.kernel.org/tip/5919b30933d7c4fac1f214c59f26c5e990044f09
Author:     Jiri Olsa <[email protected]>
AuthorDate: Tue, 19 Mar 2013 15:35:09 +0100
Committer:  Ingo Molnar <[email protected]>
CommitDate: Wed, 1 May 2013 12:34:46 +0200

perf: Fix vmalloc ring buffer pages handling

If we allocate perf ring buffer with the size of single (user) page, we will get memory corruption when releasing itin rb_free_work function (for CONFIG_PERF_USE_VMALLOC option).

For single page sized ring buffer the page_order is -1 (because nr_pages is 0). This needs to be recognized in the rb_free_work function to release proper amount of pages.

Adding data_page_nr function that returns number of allocated data pages. Customizing the rest of the code to use it.

---
 kernel/events/ring_buffer.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)
kernel/events/ring_buffer.c +10 -4
index 97fddb0..cd55144 100644
@@ -326,11 +326,16 @@ void rb_free(struct ring_buffer *rb)
 }
 
 #else
+static int data_page_nr(struct ring_buffer *rb)
+{
+	return rb->nr_pages << page_order(rb);
+}
 
 struct page *
 perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff)
 {
-	if (pgoff > (1UL << page_order(rb)))
+	/* The '>' counts in the user page. */
+	if (pgoff > data_page_nr(rb))
 		return NULL;
 
 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
@@ -350,10 +355,11 @@ static void rb_free_work(struct work_struct *work)
 	int i, nr;
 
 	rb = container_of(work, struct ring_buffer, work);
-	nr = 1 << page_order(rb);
+	nr = data_page_nr(rb);
 
 	base = rb->user_page;
-	for (i = 0; i < nr + 1; i++)
+	/* The '<=' counts in the user page. */
+	for (i = 0; i <= nr; i++)
 		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
 
 	vfree(base);
@@ -387,7 +393,7 @@ struct ring_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
 	rb->user_page = all_buf;
 	rb->data_pages[0] = all_buf + PAGE_SIZE;
 	rb->page_order = ilog2(nr_pages);
-	rb->nr_pages = 1;
+	rb->nr_pages = !!nr_pages;
 
 	ring_buffer_init(rb, watermark, flags);