mm/migrate_device: consolidate compound folio handling
Subsystems (2)
PATCH v3 1/1 mm/migrate_device: consolidate compound folio handling
Commit dc41e961a269 ("mm/migrate_device: avoid out-of-bounds writes for compound folios") added handling for compound folios that do not fit in the remaining PFN array.
migrate_device_range() and migrate_device_pfns() duplicate the logic for locking device PFNs, encoding compound folios, and handling this boundary condition.
A compound folio cannot be represented partially for migration. Use VM_WARN_ON_ONCE() when one does not fit in the remaining PFN array, while retaining the existing defensive handling: release any lock and reference acquired for the current folio, clear the remaining entries, and stop collecting.
Move the shared collection and encoding logic into a helper so both interfaces handle compound folios consistently. Also use memset() for the compound-folio tail entries instead of open-coding the clearing loop.
If locking a folio fails, leave its source entry zero while still consuming and clearing the slots belonging to the whole folio. This prevents a tail page from being treated as a new source PFN by the caller's next iteration.
Document that an encountered compound folio must fit entirely in the remaining range or PFN array.
---
Changes in v3:
- Keep the source entry zero when migrate_device_pfn_lock() fails instead of setting MIGRATE_PFN_COMPOUND, while still consuming and clearing all slots belonging to the compound folio.
- Use VM_WARN_ON_ONCE() for the truncated compound-folio invariant while keeping the defensive clear-and-stop path independent of CONFIG_DEBUG_VM, as suggested by Balbir Singh.
Changes in v2:
- Fix the helper parameter indentation and keep the declaration to two lines, as suggested by David Hildenbrand.
mm/migrate_device.c | 91 +++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 41 deletions(-)
mm/migrate_device.c +50 -41
@@ -1392,6 +1392,40 @@ static unsigned long migrate_device_pfn_lock(unsigned long pfn)return migrate_pfn (pfn) |MIGRATE_PFN_MIGRATE ; } +/* +* Collect a device folio into the page-granular PFN array. +* +* Return the number of entries consumed, or 0 if the folio does not fit in +* the remaining array. +*/ +static unsigned int migrate_device_collect_folio (unsigned long *src_pfn, +unsigned long pfn,unsigned long remaining) +{ +struct folio *folio =page_folio (pfn_to_page (pfn)); +unsigned int nr; + + *src_pfn =migrate_device_pfn_lock (pfn); + nr =folio_nr_pages (folio); + +VM_WARN_ON_ONCE (nr > remaining); +if (nr > remaining) { +if (*src_pfn &MIGRATE_PFN_MIGRATE ) { +folio_unlock (folio); +folio_put (folio); + } +memset (src_pfn,0 , remaining *sizeof (*src_pfn)); +return 0 ; + } + +if (nr >1 ) { +if (*src_pfn) + *src_pfn |=MIGRATE_PFN_COMPOUND ; +memset (src_pfn +1 ,0 , (nr -1 ) *sizeof (*src_pfn)); + } + +return nr; +} +/** * migrate_device_range() - migrate device private pfns to normal memory. * @src_pfns: array large enough to hold migrating source device private pfns.
@@ -1410,35 +1444,22 @@ static unsigned long migrate_device_pfn_lock(unsigned long pfn)* migrating pages that aren't free before unmapping them. Drivers may then * allocate destination pages and start copying data from the device to CPU * memory before calling migrate_device_pages(). +* +* A compound folio must fit entirely in the remaining range. */ int migrate_device_range (unsigned long *src_pfns,unsigned long start,unsigned long npages) { -unsigned long i, j, pfn; +unsigned long i, pfn;for (pfn = start, i =0 ; i < npages; pfn++, i++) { -struct page *page =pfn_to_page (pfn); -struct folio *folio =page_folio (page); -unsigned int nr =1 ; +unsigned int nr; - src_pfns[i] =migrate_device_pfn_lock (pfn); - nr =folio_nr_pages (folio); -if (nr > npages - i) { -if (src_pfns[i] &MIGRATE_PFN_MIGRATE ) { -folio_unlock (folio); -folio_put (folio); - } -memset (&src_pfns[i],0 , - (npages - i) *sizeof (*src_pfns)); + nr =migrate_device_collect_folio (&src_pfns[i], pfn, npages - i); +if (!nr)break ; - } -if (nr >1 ) { - src_pfns[i] |=MIGRATE_PFN_COMPOUND ; -for (j =1 ; j < nr; j++) - src_pfns[i+j] =0 ; - i += j -1 ; - pfn += j -1 ; - } + i += nr -1 ; + pfn += nr -1 ; }migrate_device_unmap (src_pfns, npages,NULL );
@@ -1454,33 +1475,21 @@ EXPORT_SYMBOL(migrate_device_range);* * Similar to migrate_device_range() but supports non-contiguous pre-populated * array of device pages to migrate. +* +* A compound folio must fit entirely in the remaining PFN array. */ int migrate_device_pfns (unsigned long *src_pfns,unsigned long npages) { -unsigned long i, j; +unsigned long i;for (i =0 ; i < npages; i++) { -struct page *page =pfn_to_page (src_pfns[i]); -struct folio *folio =page_folio (page); -unsigned int nr =1 ; +unsigned long pfn = src_pfns[i]; +unsigned int nr; - src_pfns[i] =migrate_device_pfn_lock (src_pfns[i]); - nr =folio_nr_pages (folio); -if (nr > npages - i) { -if (src_pfns[i] &MIGRATE_PFN_MIGRATE ) { -folio_unlock (folio); -folio_put (folio); - } -memset (&src_pfns[i],0 , - (npages - i) *sizeof (*src_pfns)); + nr =migrate_device_collect_folio (&src_pfns[i], pfn, npages - i); +if (!nr)break ; - } -if (nr >1 ) { - src_pfns[i] |=MIGRATE_PFN_COMPOUND ; -for (j =1 ; j < nr; j++) - src_pfns[i+j] =0 ; - i += j -1 ; - } + i += nr -1 ; }migrate_device_unmap (src_pfns, npages,NULL );
base-commit: fe2ec83746e501645709761605c2464a44fd2929
Signature · 1 line
2.55.0
David Hildenbrand (Arm)
maintainerQuoted 57 lines · Hui Su
Commit
dc41e961a269("mm/migrate_device: avoid out-of-bounds writes for compound folios") added handling for compound folios that do not fit in the remaining PFN array.
migrate_device_range()andmigrate_device_pfns()duplicate the logic for locking device PFNs, encoding compound folios, and handling this boundary condition.A compound folio cannot be represented partially for migration. Use
VM_WARN_ON_ONCE()when one does not fit in the remaining PFN array, while retaining the existing defensive handling: release any lock and reference acquired for the current folio, clear the remaining entries, and stop collecting.Move the shared collection and encoding logic into a helper so both interfaces handle compound folios consistently. Also use
memset()for the compound-folio tail entries instead of open-coding the clearing loop.If locking a folio fails, leave its source entry zero while still consuming and clearing the slots belonging to the whole folio. This prevents a tail page from being treated as a new source PFN by the caller's next iteration.
Document that an encountered compound folio must fit entirely in the remaining range or PFN array.
Suggested-by David Hildenbrand <[email protected]>Acked-by Balbir Singh <[email protected]>Signed-off-by Hui Su <[email protected]>---
Changes in v3:
- Keep the source entry zero whenmigrate_device_pfn_lock()fails instead of settingMIGRATE_PFN_COMPOUND, while still consuming and clearing all slots belonging to the compound folio.
- UseVM_WARN_ON_ONCE()for the truncated compound-folio invariant while keeping the defensive clear-and-stop path independent ofCONFIG_DEBUG_VM, as suggested by Balbir Singh.Changes in v2:
- Fix the helper parameter indentation and keep the declaration to two lines, as suggested by David Hildenbrand.mm/migrate_device.c | 91 +++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 41 deletions(-)mm/migrate_device.c +3
-0@@ -1392,6 +1392,40 @@ static unsigned long migrate_device_pfn_lock(unsigned long pfn)return migrate_pfn (pfn) |MIGRATE_PFN_MIGRATE ; } +/* +* Collect a device folio into the page-granular PFN array. +*
mm/migrate_device.c static unsigned long migrate_device_pfn_lock(unsigned long pfn) · line 1407
Quoted 10 lines
mm/migrate_device.c +10
-0+* Return the number of entries consumed, or 0 if the folio does not fit in +* the remaining array. +*/ +static unsigned int migrate_device_collect_folio (unsigned long *src_pfn, +unsigned long pfn,unsigned long remaining) +{ +struct folio *folio =page_folio (pfn_to_page (pfn)); +unsigned int nr; + + *src_pfn =migrate_device_pfn_lock (pfn);
Let's rant about migrate_device_pfn_lock().
It looks up a folio (which we do as well !?) to then get+lock that folio, but doesn't tell us about the folio, but we're supposed to magically unlock and put that folio on the backout path?.
And if it fails (e.g., folio_get_nontail_page() failed) we simply go ahead and work on that folio (folio_nr_pages() etc)?
What?!
Now that we have a single caller of migrate_device_pfn_lock(), can we please get rid of this abomination and figure out what to do when migrate_device_pfn_lock() would have returned 0?
mm/migrate_device.c static unsigned long migrate_device_pfn_lock(unsigned long pfn) · line 1411
Quoted 4 lines
mm/migrate_device.c +4
-0+ nr =folio_nr_pages (folio); + +VM_WARN_ON_ONCE (nr > remaining); +if (nr > remaining) {
It's odd to add recovery paths for something that is mostly an internal assertion. The pattern is odd.
I'd either
if (WARN_ON_ONCE(nr > remaining)) {
...
}Or only do the VM_WARN_ON_ONCE() and drop recovery, expecting this to be found early during testing.
Quoted 16 lines
mm/migrate_device.c +16
-0+if (*src_pfn &MIGRATE_PFN_MIGRATE ) { +folio_unlock (folio); +folio_put (folio); + } +memset (src_pfn,0 , remaining *sizeof (*src_pfn)); +return 0 ; + } + +if (nr >1 ) { +if (*src_pfn) + *src_pfn |=MIGRATE_PFN_COMPOUND ; +memset (src_pfn +1 ,0 , (nr -1 ) *sizeof (*src_pfn)); + } + +return nr; +}
Signature · 3 lines
Cheers, David